You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/09/30 16:09:27 UTC

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1445 - Still Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1445/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC

1 tests failed.
FAILED:  org.apache.solr.cloud.TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates

Error Message:
Can not find doc 1 in http://127.0.0.1:58840/solr

Stack Trace:
java.lang.AssertionError: Can not find doc 1 in http://127.0.0.1:58840/solr
	at __randomizedtesting.SeedInfo.seed([6047B6007A8B1ACA:E6864EED25DACC2A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.TestTlogReplica.checkRTG(TestTlogReplica.java:861)
	at org.apache.solr.cloud.TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates(TestTlogReplica.java:664)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13197 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestTlogReplica
   [junit4]   2> 2758894 INFO  (SUITE-TestTlogReplica-seed#[6047B6007A8B1ACA]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/init-core-data-001
   [junit4]   2> 2758904 WARN  (SUITE-TestTlogReplica-seed#[6047B6007A8B1ACA]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=74 numCloses=74
   [junit4]   2> 2758905 INFO  (SUITE-TestTlogReplica-seed#[6047B6007A8B1ACA]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 2758906 INFO  (SUITE-TestTlogReplica-seed#[6047B6007A8B1ACA]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2758906 INFO  (SUITE-TestTlogReplica-seed#[6047B6007A8B1ACA]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001
   [junit4]   2> 2758906 INFO  (SUITE-TestTlogReplica-seed#[6047B6007A8B1ACA]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2758906 INFO  (Thread-8658) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2758907 INFO  (Thread-8658) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2758908 ERROR (Thread-8658) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2759007 INFO  (SUITE-TestTlogReplica-seed#[6047B6007A8B1ACA]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:40116
   [junit4]   2> 2759016 INFO  (jetty-launcher-3953-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2759016 INFO  (jetty-launcher-3953-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2759018 INFO  (jetty-launcher-3953-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@589407fa{/solr,null,AVAILABLE}
   [junit4]   2> 2759018 INFO  (jetty-launcher-3953-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6a41d176{/solr,null,AVAILABLE}
   [junit4]   2> 2759019 INFO  (jetty-launcher-3953-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6034cd8d{HTTP/1.1,[http/1.1]}{127.0.0.1:62246}
   [junit4]   2> 2759019 INFO  (jetty-launcher-3953-thread-1) [    ] o.e.j.s.Server Started @2761232ms
   [junit4]   2> 2759020 INFO  (jetty-launcher-3953-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=62246}
   [junit4]   2> 2759026 ERROR (jetty-launcher-3953-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2759026 INFO  (jetty-launcher-3953-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 2759027 INFO  (jetty-launcher-3953-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2759027 INFO  (jetty-launcher-3953-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2759027 INFO  (jetty-launcher-3953-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-30T15:39:27.734Z
   [junit4]   2> 2759040 INFO  (jetty-launcher-3953-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@345604b3{HTTP/1.1,[http/1.1]}{127.0.0.1:58840}
   [junit4]   2> 2759040 INFO  (jetty-launcher-3953-thread-2) [    ] o.e.j.s.Server Started @2761253ms
   [junit4]   2> 2759040 INFO  (jetty-launcher-3953-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=58840}
   [junit4]   2> 2759040 ERROR (jetty-launcher-3953-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2759041 INFO  (jetty-launcher-3953-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 2759041 INFO  (jetty-launcher-3953-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2759041 INFO  (jetty-launcher-3953-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2759041 INFO  (jetty-launcher-3953-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-30T15:39:27.748Z
   [junit4]   2> 2759044 INFO  (jetty-launcher-3953-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2759051 INFO  (jetty-launcher-3953-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2759054 INFO  (jetty-launcher-3953-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40116/solr
   [junit4]   2> 2759066 INFO  (jetty-launcher-3953-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40116/solr
   [junit4]   2> 2759171 INFO  (jetty-launcher-3953-thread-2) [n:127.0.0.1:58840_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2759172 INFO  (jetty-launcher-3953-thread-2) [n:127.0.0.1:58840_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:58840_solr
   [junit4]   2> 2759180 INFO  (jetty-launcher-3953-thread-2) [n:127.0.0.1:58840_solr    ] o.a.s.c.Overseer Overseer (id=98748725173551110-127.0.0.1:58840_solr-n_0000000000) starting
   [junit4]   2> 2759206 INFO  (jetty-launcher-3953-thread-2) [n:127.0.0.1:58840_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58840_solr
   [junit4]   2> 2759274 INFO  (zkCallback-3966-thread-1-processing-n:127.0.0.1:58840_solr) [n:127.0.0.1:58840_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2759344 INFO  (jetty-launcher-3953-thread-2) [n:127.0.0.1:58840_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58840.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@12b93636
   [junit4]   2> 2759352 INFO  (jetty-launcher-3953-thread-2) [n:127.0.0.1:58840_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58840.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@12b93636
   [junit4]   2> 2759352 INFO  (jetty-launcher-3953-thread-2) [n:127.0.0.1:58840_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58840.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@12b93636
   [junit4]   2> 2759354 INFO  (jetty-launcher-3953-thread-2) [n:127.0.0.1:58840_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001/node2/.
   [junit4]   2> 2759469 INFO  (jetty-launcher-3953-thread-1) [n:127.0.0.1:62246_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2759473 INFO  (jetty-launcher-3953-thread-1) [n:127.0.0.1:62246_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2759487 INFO  (jetty-launcher-3953-thread-1) [n:127.0.0.1:62246_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62246_solr
   [junit4]   2> 2759489 INFO  (zkCallback-3966-thread-1-processing-n:127.0.0.1:58840_solr) [n:127.0.0.1:58840_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2759508 INFO  (zkCallback-3961-thread-1-processing-n:127.0.0.1:62246_solr) [n:127.0.0.1:62246_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2759789 INFO  (jetty-launcher-3953-thread-1) [n:127.0.0.1:62246_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62246.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@12b93636
   [junit4]   2> 2759815 INFO  (jetty-launcher-3953-thread-1) [n:127.0.0.1:62246_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62246.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@12b93636
   [junit4]   2> 2759816 INFO  (jetty-launcher-3953-thread-1) [n:127.0.0.1:62246_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62246.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@12b93636
   [junit4]   2> 2759817 INFO  (jetty-launcher-3953-thread-1) [n:127.0.0.1:62246_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001/node1/.
   [junit4]   2> 2759904 INFO  (SUITE-TestTlogReplica-seed#[6047B6007A8B1ACA]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2759905 INFO  (SUITE-TestTlogReplica-seed#[6047B6007A8B1ACA]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40116/solr ready
   [junit4]   2> 2759909 INFO  (SUITE-TestTlogReplica-seed#[6047B6007A8B1ACA]-worker) [    ] o.a.s.c.TestTlogReplica Using legacyCloud?: false
   [junit4]   2> 2759911 INFO  (qtp280744521-26951) [n:127.0.0.1:58840_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2759917 INFO  (qtp280744521-26951) [n:127.0.0.1:58840_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=6
   [junit4]   2> 2759942 INFO  (TEST-TestTlogReplica.testCreateDelete-seed#[6047B6007A8B1ACA]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testCreateDelete {seed=[6047B6007A8B1ACA:7B57474C8C7BE2FA]}
   [junit4]   2> 2759953 INFO  (OverseerThreadFactory-9638-thread-1-processing-n:127.0.0.1:58840_solr) [n:127.0.0.1:58840_solr    ] o.a.s.c.CreateCollectionCmd Create collection tlog_replica_test_create_delete
   [junit4]   2> 2759954 WARN  (OverseerThreadFactory-9638-thread-1-processing-n:127.0.0.1:58840_solr) [n:127.0.0.1:58840_solr    ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 4 on collection tlog_replica_test_create_delete is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance.
   [junit4]   2> 2760070 INFO  (OverseerStateUpdate-98748725173551110-127.0.0.1:58840_solr-n_0000000000) [n:127.0.0.1:58840_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"tlog_replica_test_create_delete",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"tlog_replica_test_create_delete_shard1_replica_t1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:58840/solr",
   [junit4]   2>   "type":"TLOG"} 
   [junit4]   2> 2760074 INFO  (OverseerStateUpdate-98748725173551110-127.0.0.1:58840_solr-n_0000000000) [n:127.0.0.1:58840_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"tlog_replica_test_create_delete",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"tlog_replica_test_create_delete_shard1_replica_t2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:62246/solr",
   [junit4]   2>   "type":"TLOG"} 
   [junit4]   2> 2760079 INFO  (OverseerStateUpdate-98748725173551110-127.0.0.1:58840_solr-n_0000000000) [n:127.0.0.1:58840_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"tlog_replica_test_create_delete",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"tlog_replica_test_create_delete_shard1_replica_t4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:58840/solr",
   [junit4]   2>   "type":"TLOG"} 
   [junit4]   2> 2760082 INFO  (OverseerStateUpdate-98748725173551110-127.0.0.1:58840_solr-n_0000000000) [n:127.0.0.1:58840_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"tlog_replica_test_create_delete",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"tlog_replica_test_create_delete_shard1_replica_t6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:62246/solr",
   [junit4]   2>   "type":"TLOG"} 
   [junit4]   2> 2760085 INFO  (OverseerStateUpdate-98748725173551110-127.0.0.1:58840_solr-n_0000000000) [n:127.0.0.1:58840_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"tlog_replica_test_create_delete",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"tlog_replica_test_create_delete_shard2_replica_t8",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:58840/solr",
   [junit4]   2>   "type":"TLOG"} 
   [junit4]   2> 2760089 INFO  (OverseerStateUpdate-98748725173551110-127.0.0.1:58840_solr-n_0000000000) [n:127.0.0.1:58840_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"tlog_replica_test_create_delete",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"tlog_replica_test_create_delete_shard2_replica_t10",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:62246/solr",
   [junit4]   2>   "type":"TLOG"} 
   [junit4]   2> 2760098 INFO  (OverseerStateUpdate-98748725173551110-127.0.0.1:58840_solr-n_0000000000) [n:127.0.0.1:58840_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"tlog_replica_test_create_delete",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"tlog_replica_test_create_delete_shard2_replica_t12",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:58840/solr",
   [junit4]   2>   "type":"TLOG"} 
   [junit4]   2> 2760104 INFO  (OverseerStateUpdate-98748725173551110-127.0.0.1:58840_solr-n_0000000000) [n:127.0.0.1:58840_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"tlog_replica_test_create_delete",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"tlog_replica_test_create_delete_shard2_replica_t14",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:62246/solr",
   [junit4]   2>   "type":"TLOG"} 
   [junit4]   2> 2760307 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=tlog_replica_test_create_delete_shard1_replica_t4&action=CREATE&numShards=2&collection=tlog_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2760308 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2760308 INFO  (qtp280744521-26954) [n:127.0.0.1:58840_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=tlog_replica_test_create_delete_shard1_replica_t1&action=CREATE&numShards=2&collection=tlog_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2760308 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=tlog_replica_test_create_delete_shard1_replica_t2&action=CREATE&numShards=2&collection=tlog_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2760308 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2760309 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node11&collection.configName=conf&newCollection=true&name=tlog_replica_test_create_delete_shard2_replica_t8&action=CREATE&numShards=2&collection=tlog_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2760309 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node15&collection.configName=conf&newCollection=true&name=tlog_replica_test_create_delete_shard2_replica_t12&action=CREATE&numShards=2&collection=tlog_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2760309 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node16&collection.configName=conf&newCollection=true&name=tlog_replica_test_create_delete_shard2_replica_t14&action=CREATE&numShards=2&collection=tlog_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2760309 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node9&collection.configName=conf&newCollection=true&name=tlog_replica_test_create_delete_shard1_replica_t6&action=CREATE&numShards=2&collection=tlog_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2760309 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node13&collection.configName=conf&newCollection=true&name=tlog_replica_test_create_delete_shard2_replica_t10&action=CREATE&numShards=2&collection=tlog_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 2760435 INFO  (zkCallback-3966-thread-1-processing-n:127.0.0.1:58840_solr) [n:127.0.0.1:58840_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_create_delete/state.json] for collection [tlog_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2760435 INFO  (zkCallback-3966-thread-2-processing-n:127.0.0.1:58840_solr) [n:127.0.0.1:58840_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_create_delete/state.json] for collection [tlog_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2760435 INFO  (zkCallback-3961-thread-1-processing-n:127.0.0.1:62246_solr) [n:127.0.0.1:62246_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_create_delete/state.json] for collection [tlog_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2760435 INFO  (zkCallback-3961-thread-2-processing-n:127.0.0.1:62246_solr) [n:127.0.0.1:62246_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_create_delete/state.json] for collection [tlog_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2761359 INFO  (qtp280744521-26954) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node3 x:tlog_replica_test_create_delete_shard1_replica_t1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2761362 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2761362 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2761368 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2761370 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2761371 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.s.IndexSchema [tlog_replica_test_create_delete_shard2_replica_t14] Schema name=minimal
   [junit4]   2> 2761374 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2761374 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_create_delete_shard2_replica_t14' using configuration from collection tlog_replica_test_create_delete, trusted=true
   [junit4]   2> 2761375 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62246.solr.core.tlog_replica_test_create_delete.shard2.replica_t14' (registry 'solr.core.tlog_replica_test_create_delete.shard2.replica_t14') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@12b93636
   [junit4]   2> 2761375 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2761375 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.c.SolrCore [[tlog_replica_test_create_delete_shard2_replica_t14] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001/node1/tlog_replica_test_create_delete_shard2_replica_t14], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001/node1/./tlog_replica_test_create_delete_shard2_replica_t14/data/]
   [junit4]   2> 2761376 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2761378 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2761390 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.s.IndexSchema [tlog_replica_test_create_delete_shard1_replica_t6] Schema name=minimal
   [junit4]   2> 2761390 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2761392 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2761392 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_create_delete_shard1_replica_t6' using configuration from collection tlog_replica_test_create_delete, trusted=true
   [junit4]   2> 2761393 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62246.solr.core.tlog_replica_test_create_delete.shard1.replica_t6' (registry 'solr.core.tlog_replica_test_create_delete.shard1.replica_t6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@12b93636
   [junit4]   2> 2761393 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2761393 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.c.SolrCore [[tlog_replica_test_create_delete_shard1_replica_t6] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001/node1/tlog_replica_test_create_delete_shard1_replica_t6], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001/node1/./tlog_replica_test_create_delete_shard1_replica_t6/data/]
   [junit4]   2> 2761395 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.s.IndexSchema [tlog_replica_test_create_delete_shard2_replica_t10] Schema name=minimal
   [junit4]   2> 2761396 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2761396 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_create_delete_shard2_replica_t10' using configuration from collection tlog_replica_test_create_delete, trusted=true
   [junit4]   2> 2761396 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62246.solr.core.tlog_replica_test_create_delete.shard2.replica_t10' (registry 'solr.core.tlog_replica_test_create_delete.shard2.replica_t10') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@12b93636
   [junit4]   2> 2761397 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2761397 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.c.SolrCore [[tlog_replica_test_create_delete_shard2_replica_t10] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001/node1/tlog_replica_test_create_delete_shard2_replica_t10], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001/node1/./tlog_replica_test_create_delete_shard2_replica_t10/data/]
   [junit4]   2> 2761397 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.s.IndexSchema [tlog_replica_test_create_delete_shard1_replica_t2] Schema name=minimal
   [junit4]   2> 2761399 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2761399 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_create_delete_shard1_replica_t2' using configuration from collection tlog_replica_test_create_delete, trusted=true
   [junit4]   2> 2761399 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62246.solr.core.tlog_replica_test_create_delete.shard1.replica_t2' (registry 'solr.core.tlog_replica_test_create_delete.shard1.replica_t2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@12b93636
   [junit4]   2> 2761399 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2761399 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.c.SolrCore [[tlog_replica_test_create_delete_shard1_replica_t2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001/node1/tlog_replica_test_create_delete_shard1_replica_t2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001/node1/./tlog_replica_test_create_delete_shard1_replica_t2/data/]
   [junit4]   2> 2761406 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.s.IndexSchema [tlog_replica_test_create_delete_shard2_replica_t8] Schema name=minimal
   [junit4]   2> 2761407 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2761407 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_create_delete_shard2_replica_t8' using configuration from collection tlog_replica_test_create_delete, trusted=true
   [junit4]   2> 2761408 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58840.solr.core.tlog_replica_test_create_delete.shard2.replica_t8' (registry 'solr.core.tlog_replica_test_create_delete.shard2.replica_t8') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@12b93636
   [junit4]   2> 2761408 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2761408 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.c.SolrCore [[tlog_replica_test_create_delete_shard2_replica_t8] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001/node2/tlog_replica_test_create_delete_shard2_replica_t8], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001/node2/./tlog_replica_test_create_delete_shard2_replica_t8/data/]
   [junit4]   2> 2761411 INFO  (qtp280744521-26954) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node3 x:tlog_replica_test_create_delete_shard1_replica_t1] o.a.s.s.IndexSchema [tlog_replica_test_create_delete_shard1_replica_t1] Schema name=minimal
   [junit4]   2> 2761412 INFO  (qtp280744521-26954) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node3 x:tlog_replica_test_create_delete_shard1_replica_t1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2761412 INFO  (qtp280744521-26954) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node3 x:tlog_replica_test_create_delete_shard1_replica_t1] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_create_delete_shard1_replica_t1' using configuration from collection tlog_replica_test_create_delete, trusted=true
   [junit4]   2> 2761413 INFO  (qtp280744521-26954) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node3 x:tlog_replica_test_create_delete_shard1_replica_t1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58840.solr.core.tlog_replica_test_create_delete.shard1.replica_t1' (registry 'solr.core.tlog_replica_test_create_delete.shard1.replica_t1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@12b93636
   [junit4]   2> 2761413 INFO  (qtp280744521-26954) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node3 x:tlog_replica_test_create_delete_shard1_replica_t1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2761413 INFO  (qtp280744521-26954) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node3 x:tlog_replica_test_create_delete_shard1_replica_t1] o.a.s.c.SolrCore [[tlog_replica_test_create_delete_shard1_replica_t1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001/node2/tlog_replica_test_create_delete_shard1_replica_t1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001/node2/./tlog_replica_test_create_delete_shard1_replica_t1/data/]
   [junit4]   2> 2761416 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.s.IndexSchema [tlog_replica_test_create_delete_shard1_replica_t4] Schema name=minimal
   [junit4]   2> 2761417 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2761418 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_create_delete_shard1_replica_t4' using configuration from collection tlog_replica_test_create_delete, trusted=true
   [junit4]   2> 2761418 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58840.solr.core.tlog_replica_test_create_delete.shard1.replica_t4' (registry 'solr.core.tlog_replica_test_create_delete.shard1.replica_t4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@12b93636
   [junit4]   2> 2761418 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2761418 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.c.SolrCore [[tlog_replica_test_create_delete_shard1_replica_t4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001/node2/tlog_replica_test_create_delete_shard1_replica_t4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001/node2/./tlog_replica_test_create_delete_shard1_replica_t4/data/]
   [junit4]   2> 2761424 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.s.IndexSchema [tlog_replica_test_create_delete_shard2_replica_t12] Schema name=minimal
   [junit4]   2> 2761432 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2761432 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.c.CoreContainer Creating SolrCore 'tlog_replica_test_create_delete_shard2_replica_t12' using configuration from collection tlog_replica_test_create_delete, trusted=true
   [junit4]   2> 2761432 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58840.solr.core.tlog_replica_test_create_delete.shard2.replica_t12' (registry 'solr.core.tlog_replica_test_create_delete.shard2.replica_t12') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@12b93636
   [junit4]   2> 2761432 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2761432 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.c.SolrCore [[tlog_replica_test_create_delete_shard2_replica_t12] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001/node2/tlog_replica_test_create_delete_shard2_replica_t12], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001/tempDir-001/node2/./tlog_replica_test_create_delete_shard2_replica_t12/data/]
   [junit4]   2> 2761474 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2761475 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2761476 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2761476 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2761477 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.s.SolrIndexSearcher Opening [Searcher@6b3b9201[tlog_replica_test_create_delete_shard2_replica_t10] main]
   [junit4]   2> 2761480 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 2761480 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 2761481 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2761481 INFO  (searcherExecutor-9645-thread-1-processing-n:127.0.0.1:62246_solr x:tlog_replica_test_create_delete_shard2_replica_t10 s:shard2 c:tlog_replica_test_create_delete r:core_node13) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.c.SolrCore [tlog_replica_test_create_delete_shard2_replica_t10] Registered new searcher Searcher@6b3b9201[tlog_replica_test_create_delete_shard2_replica_t10] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2761482 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1579979605476900864
   [junit4]   2> 2761488 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2761489 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2761490 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2761490 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2761491 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.s.SolrIndexSearcher Opening [Searcher@2d080a0c[tlog_replica_test_create_delete_shard1_replica_t6] main]
   [junit4]   2> 2761492 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard2: total=4 found=1 timeoutin=9999ms
   [junit4]   2> 2761493 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 2761493 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 2761493 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2761494 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1579979605489483776
   [junit4]   2> 2761494 INFO  (searcherExecutor-9644-thread-1-processing-n:127.0.0.1:62246_solr x:tlog_replica_test_create_delete_shard1_replica_t6 s:shard1 c:tlog_replica_test_create_delete r:core_node9) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.c.SolrCore [tlog_replica_test_create_delete_shard1_replica_t6] Registered new searcher Searcher@2d080a0c[tlog_replica_test_create_delete_shard1_replica_t6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2761499 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=4 found=1 timeoutin=9999ms
   [junit4]   2> 2761519 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2761520 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2761521 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2761521 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2761522 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.s.SolrIndexSearcher Opening [Searcher@4fb8184a[tlog_replica_test_create_delete_shard1_replica_t2] main]
   [junit4]   2> 2761529 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 2761530 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 2761534 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2761534 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1579979605531426816
   [junit4]   2> 2761534 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2761535 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2761539 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2761540 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2761543 INFO  (searcherExecutor-9646-thread-1-processing-n:127.0.0.1:62246_solr x:tlog_replica_test_create_delete_shard1_replica_t2 s:shard1 c:tlog_replica_test_create_delete r:core_node5) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.c.SolrCore [tlog_replica_test_create_delete_shard1_replica_t2] Registered new searcher Searcher@4fb8184a[tlog_replica_test_create_delete_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2761548 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.s.SolrIndexSearcher Opening [Searcher@250570c6[tlog_replica_test_create_delete_shard1_replica_t4] main]
   [junit4]   2> 2761549 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 2761549 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 2761552 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2761552 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1579979605550301184
   [junit4]   2> 2761554 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2761555 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2761558 INFO  (searcherExecutor-9649-thread-1-processing-n:127.0.0.1:58840_solr x:tlog_replica_test_create_delete_shard1_replica_t4 s:shard1 c:tlog_replica_test_create_delete r:core_node7) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.c.SolrCore [tlog_replica_test_create_delete_shard1_replica_t4] Registered new searcher Searcher@250570c6[tlog_replica_test_create_delete_shard1_replica_t4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2761559 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2761559 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2761564 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.s.SolrIndexSearcher Opening [Searcher@a8f6456[tlog_replica_test_create_delete_shard2_replica_t8] main]
   [junit4]   2> 2761565 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 2761565 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 2761568 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2761573 INFO  (searcherExecutor-9647-thread-1-processing-n:127.0.0.1:58840_solr x:tlog_replica_test_create_delete_shard2_replica_t8 s:shard2 c:tlog_replica_test_create_delete r:core_node11) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.c.SolrCore [tlog_replica_test_create_delete_shard2_replica_t8] Registered new searcher Searcher@a8f6456[tlog_replica_test_create_delete_shard2_replica_t8] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2761573 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1579979605572321280
   [junit4]   2> 2761580 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2761580 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2761582 INFO  (qtp280744521-26954) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node3 x:tlog_replica_test_create_delete_shard1_replica_t1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2761582 INFO  (qtp280744521-26954) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node3 x:tlog_replica_test_create_delete_shard1_replica_t1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2761582 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2761582 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2761582 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2761582 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2761583 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.s.SolrIndexSearcher Opening [Searcher@25a787a1[tlog_replica_test_create_delete_shard2_replica_t14] main]
   [junit4]   2> 2761583 INFO  (qtp280744521-26954) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node3 x:tlog_replica_test_create_delete_shard1_replica_t1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2761584 INFO  (qtp280744521-26954) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node3 x:tlog_replica_test_create_delete_shard1_replica_t1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2761584 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2761584 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2761584 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 2761584 INFO  (qtp280744521-26954) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node3 x:tlog_replica_test_create_delete_shard1_replica_t1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6a62a202[tlog_replica_test_create_delete_shard1_replica_t1] main]
   [junit4]   2> 2761585 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 2761585 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.s.SolrIndexSearcher Opening [Searcher@7c5e51fc[tlog_replica_test_create_delete_shard2_replica_t12] main]
   [junit4]   2> 2761586 INFO  (qtp280744521-26954) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node3 x:tlog_replica_test_create_delete_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 2761586 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2761586 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1579979605585952768
   [junit4]   2> 2761586 INFO  (qtp280744521-26954) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node3 x:tlog_replica_test_create_delete_shard1_replica_t1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 2761587 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 2761587 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 2761587 INFO  (searcherExecutor-9643-thread-1-processing-n:127.0.0.1:62246_solr x:tlog_replica_test_create_delete_shard2_replica_t14 s:shard2 c:tlog_replica_test_create_delete r:core_node16) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.c.SolrCore [tlog_replica_test_create_delete_shard2_replica_t14] Registered new searcher Searcher@25a787a1[tlog_replica_test_create_delete_shard2_replica_t14] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2761587 INFO  (qtp280744521-26954) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node3 x:tlog_replica_test_create_delete_shard1_replica_t1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2761587 INFO  (qtp280744521-26954) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node3 x:tlog_replica_test_create_delete_shard1_replica_t1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1579979605587001344
   [junit4]   2> 2761587 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2761587 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1579979605587001344
   [junit4]   2> 2761588 INFO  (searcherExecutor-9650-thread-1-processing-n:127.0.0.1:58840_solr x:tlog_replica_test_create_delete_shard2_replica_t12 s:shard2 c:tlog_replica_test_create_delete r:core_node15) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.c.SolrCore [tlog_replica_test_create_delete_shard2_replica_t12] Registered new searcher Searcher@7c5e51fc[tlog_replica_test_create_delete_shard2_replica_t12] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2761588 INFO  (searcherExecutor-9648-thread-1-processing-n:127.0.0.1:58840_solr x:tlog_replica_test_create_delete_shard1_replica_t1 s:shard1 c:tlog_replica_test_create_delete r:core_node3) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node3 x:tlog_replica_test_create_delete_shard1_replica_t1] o.a.s.c.SolrCore [tlog_replica_test_create_delete_shard1_replica_t1] Registered new searcher Searcher@6a62a202[tlog_replica_test_create_delete_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2761602 INFO  (zkCallback-3961-thread-1-processing-n:127.0.0.1:62246_solr) [n:127.0.0.1:62246_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_create_delete/state.json] for collection [tlog_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2761602 INFO  (zkCallback-3966-thread-1-processing-n:127.0.0.1:58840_solr) [n:127.0.0.1:58840_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_create_delete/state.json] for collection [tlog_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2761602 INFO  (zkCallback-3966-thread-2-processing-n:127.0.0.1:58840_solr) [n:127.0.0.1:58840_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_create_delete/state.json] for collection [tlog_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2761602 INFO  (zkCallback-3961-thread-2-processing-n:127.0.0.1:62246_solr) [n:127.0.0.1:62246_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_create_delete/state.json] for collection [tlog_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2761994 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2761994 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2761995 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:62246/solr/tlog_replica_test_create_delete_shard2_replica_t10/
   [junit4]   2> 2761995 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.u.PeerSync PeerSync: core=tlog_replica_test_create_delete_shard2_replica_t10 url=http://127.0.0.1:62246/solr START replicas=[http://127.0.0.1:58840/solr/tlog_replica_test_create_delete_shard2_replica_t8/, http://127.0.0.1:58840/solr/tlog_replica_test_create_delete_shard2_replica_t12/, http://127.0.0.1:62246/solr/tlog_replica_test_create_delete_shard2_replica_t14/] nUpdates=100
   [junit4]   2> 2761998 INFO  (qtp280744521-26947) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.c.S.Request [tlog_replica_test_create_delete_shard2_replica_t8]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2761998 INFO  (qtp280744521-26947) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.c.S.Request [tlog_replica_test_create_delete_shard2_replica_t12]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2761999 INFO  (qtp1201318454-26944) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.c.S.Request [tlog_replica_test_create_delete_shard2_replica_t14]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2762000 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2762000 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2762000 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:62246/solr/tlog_replica_test_create_delete_shard1_replica_t6/
   [junit4]   2> 2762001 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.u.PeerSync PeerSync: core=tlog_replica_test_create_delete_shard1_replica_t6 url=http://127.0.0.1:62246/solr START replicas=[http://127.0.0.1:58840/solr/tlog_replica_test_create_delete_shard1_replica_t1/, http://127.0.0.1:62246/solr/tlog_replica_test_create_delete_shard1_replica_t2/, http://127.0.0.1:58840/solr/tlog_replica_test_create_delete_shard1_replica_t4/] nUpdates=100
   [junit4]   2> 2762002 INFO  (qtp280744521-26951) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.c.S.Request [tlog_replica_test_create_delete_shard1_replica_t4]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2762002 INFO  (qtp1201318454-26946) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.c.S.Request [tlog_replica_test_create_delete_shard1_replica_t2]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2762003 INFO  (qtp280744521-27030) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node3 x:tlog_replica_test_create_delete_shard1_replica_t1] o.a.s.c.S.Request [tlog_replica_test_create_delete_shard1_replica_t1]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2762297 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.u.PeerSync PeerSync: core=tlog_replica_test_create_delete_shard2_replica_t10 url=http://127.0.0.1:62246/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 2762297 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 2762297 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 2762297 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2762298 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.c.ZkController tlog_replica_test_create_delete_shard2_replica_t10 stopping background replication from leader
   [junit4]   2> 2762301 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.u.PeerSync PeerSync: core=tlog_replica_test_create_delete_shard1_replica_t6 url=http://127.0.0.1:62246/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 2762301 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 2762301 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 2762302 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2762303 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.c.ZkController tlog_replica_test_create_delete_shard1_replica_t6 stopping background replication from leader
   [junit4]   2> 2762308 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:62246/solr/tlog_replica_test_create_delete_shard2_replica_t10/ shard2
   [junit4]   2> 2762311 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:62246/solr/tlog_replica_test_create_delete_shard1_replica_t6/ shard1
   [junit4]   2> 2762420 INFO  (zkCallback-3966-thread-2-processing-n:127.0.0.1:58840_solr) [n:127.0.0.1:58840_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_create_delete/state.json] for collection [tlog_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2762420 INFO  (zkCallback-3961-thread-1-processing-n:127.0.0.1:62246_solr) [n:127.0.0.1:62246_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_create_delete/state.json] for collection [tlog_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2762420 INFO  (zkCallback-3966-thread-1-processing-n:127.0.0.1:58840_solr) [n:127.0.0.1:58840_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_create_delete/state.json] for collection [tlog_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2762420 INFO  (zkCallback-3961-thread-2-processing-n:127.0.0.1:62246_solr) [n:127.0.0.1:62246_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tlog_replica_test_create_delete/state.json] for collection [tlog_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2762460 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2762463 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2762463 INFO  (qtp1201318454-26943) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node13 x:tlog_replica_test_create_delete_shard2_replica_t10] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node13&collection.configName=conf&newCollection=true&name=tlog_replica_test_create_delete_shard2_replica_t10&action=CREATE&numShards=2&collection=tlog_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2154
   [junit4]   2> 2762466 INFO  (qtp1201318454-26942) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node9 x:tlog_replica_test_create_delete_shard1_replica_t6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node9&collection.configName=conf&newCollection=true&name=tlog_replica_test_create_delete_shard1_replica_t6&action=CREATE&numShards=2&collection=tlog_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2157
   [junit4]   2> 2762541 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.c.ZkController tlog_replica_test_create_delete_shard1_replica_t2 starting background replication from leader
   [junit4]   2> 2762541 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 2762543 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 2762544 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2762545 INFO  (indexFetcher-9684-thread-1) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.h.IndexFetcher Replica core_node9 is leader but it's state is down, skipping replication
   [junit4]   2> 2762549 INFO  (qtp1201318454-26939) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard1 r:core_node5 x:tlog_replica_test_create_delete_shard1_replica_t2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=tlog_replica_test_create_delete_shard1_replica_t2&action=CREATE&numShards=2&collection=tlog_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2241
   [junit4]   2> 2762559 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.c.ZkController tlog_replica_test_create_delete_shard1_replica_t4 starting background replication from leader
   [junit4]   2> 2762559 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 2762561 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 2762561 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2762564 INFO  (qtp280744521-26953) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard1 r:core_node7 x:tlog_replica_test_create_delete_shard1_replica_t4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=tlog_replica_test_create_delete_shard1_replica_t4&action=CREATE&numShards=2&collection=tlog_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2257
   [junit4]   2> 2762578 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.c.ZkController tlog_replica_test_create_delete_shard2_replica_t8 starting background replication from leader
   [junit4]   2> 2762579 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 2762580 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 2762580 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2762582 INFO  (qtp280744521-26948) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node11 x:tlog_replica_test_create_delete_shard2_replica_t8] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node11&collection.configName=conf&newCollection=true&name=tlog_replica_test_create_delete_shard2_replica_t8&action=CREATE&numShards=2&collection=tlog_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2273
   [junit4]   2> 2762593 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.c.ZkController tlog_replica_test_create_delete_shard2_replica_t14 starting background replication from leader
   [junit4]   2> 2762593 INFO  (qtp1201318454-26941) [n:127.0.0.1:62246_solr c:tlog_replica_test_create_delete s:shard2 r:core_node16 x:tlog_replica_test_create_delete_shard2_replica_t14] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 2762593 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr c:tlog_replica_test_create_delete s:shard2 r:core_node15 x:tlog_replica_test_create_delete_shard2_replica_t12] o.a.s.c.ZkController tlog_replica_test_create_delete_shard2_replica_t12 starting background replication from leader
   [junit4]   2> 2762593 INFO  (qtp280744521-26950) [n:127.0.0.1:58840_solr c:tlog_r

[...truncated too long message...]

2871868 INFO  (jetty-closer-3954-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@23be711c{/solr,null,UNAVAILABLE}
   [junit4]   2> 2871872 ERROR (SUITE-TestTlogReplica-seed#[6047B6007A8B1ACA]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2871873 INFO  (SUITE-TestTlogReplica-seed#[6047B6007A8B1ACA]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40116 40116
   [junit4]   2> 2881945 INFO  (Thread-8658) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40116 40116
   [junit4]   2> 2881948 WARN  (Thread-8658) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	34	/solr/configs/conf
   [junit4]   2> 	10	/solr/aliases.json
   [junit4]   2> 	9	/solr/security.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	36	/solr/collections/tlog_replica_test_recovery/state.json
   [junit4]   2> 	29	/solr/collections/tlog_replica_test_remove_leader/state.json
   [junit4]   2> 	26	/solr/collections/tlog_replica_test_kill_leader/state.json
   [junit4]   2> 	25	/solr/collections/tlog_replica_test_kill_tlog_replica/state.json
   [junit4]   2> 	23	/solr/collections/tlog_replica_test_basic_leader_election/state.json
   [junit4]   2> 	23	/solr/collections/tlog_replica_test_add_remove_tlog_replica/state.json
   [junit4]   2> 	22	/solr/collections/tlog_replica_test_create_delete/state.json
   [junit4]   2> 	21	/solr/collections/tlog_replica_test_out_of_order_db_qwith_in_place_updates/state.json
   [junit4]   2> 	21	/solr/collections/tlog_replica_test_real_time_get/state.json
   [junit4]   2> 	15	/solr/collections/tlog_replica_test_only_leader_indexes/state.json
   [junit4]   2> 	15	/solr/collections/tlog_replica_test_delete_by_id/state.json
   [junit4]   2> 	13	/solr/collections/tlog_replica_test_add_docs/state.json
   [junit4]   2> 	10	/solr/clusterstate.json
   [junit4]   2> 	10	/solr/clusterprops.json
   [junit4]   2> 	4	/solr/overseer_elect/election/98748725173551110-127.0.0.1:58840_solr-n_0000000000
   [junit4]   2> 	4	/solr/collections/tlog_replica_test_recovery/leader_elect/shard1/election/98748725173551110-core_node3-n_0000000000
   [junit4]   2> 	3	/solr/collections/tlog_replica_test_create_delete/leader_elect/shard1/election/98748725173551109-core_node9-n_0000000000
   [junit4]   2> 	2	/solr/overseer_elect/election/98748725173551127-127.0.0.1:62246_solr-n_0000000006
   [junit4]   2> 	2	/solr/collections/tlog_replica_test_create_delete/leader_elect/shard2/election/98748725173551109-core_node13-n_0000000000
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	10	/solr/collections
   [junit4]   2> 	4	/solr/overseer/collection-queue-work
   [junit4]   2> 	3	/solr/overseer/queue
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestTlogReplica_6047B6007A8B1ACA-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {foo=PostingsFormat(name=LuceneVarGapFixedInterval), id=FSTOrd50, foo_s=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{_version_=DocValuesFormat(name=Memory), id=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=1190, maxMBSortInHeap=7.325327421786694, sim=RandomSimilarity(queryNorm=true): {}, locale=fr-BE, timezone=Europe/Helsinki
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=235331336,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [SpellCheckComponentTest, TestSolr4Spatial, TestRecoveryHdfs, DOMUtilTest, SolrPluginUtilsTest, SolrCloudReportersTest, BlockCacheTest, TestMergePolicyConfig, TestManagedResourceStorage, TestNestedDocsSort, DistributedFacetPivotLongTailTest, ZkSolrClientTest, UpdateRequestProcessorFactoryTest, TestSerializedLuceneMatchVersion, TestInPlaceUpdatesStandalone, TestBulkSchemaConcurrent, DistributedQueryElevationComponentTest, HdfsWriteToMultipleCollectionsTest, DocValuesMultiTest, TestNonDefinedSimilarityFactory, TestRandomFlRTGCloud, FieldMutatingUpdateProcessorTest, SolrGraphiteReporterTest, SimpleCollectionCreateDeleteTest, SegmentsInfoRequestHandlerTest, LeaderInitiatedRecoveryOnCommitTest, HdfsSyncSliceTest, MoveReplicaTest, TestConfigOverlay, TestJoin, TestLMJelinekMercerSimilarityFactory, TestBlobHandler, CopyFieldTest, TestPerFieldSimilarity, SuggestComponentContextFilterQueryTest, TestTrackingShardHandlerFactory, TestConfig, QueryEqualityTest, LeaderFailoverAfterPartitionTest, TestManagedStopFilterFactory, TestSimpleTrackingShardHandler, SolrRequestParserTest, TestSolrDeletionPolicy1, OverseerModifyCollectionTest, DocumentBuilderTest, SuggesterTSTTest, TestHdfsBackupRestoreCore, TestNRTOpen, TestSchemalessBufferedUpdates, TestShortCircuitedRequests, TestScoreJoinQPScore, TestSystemIdResolver, TestTestInjection, TestBadConfig, CreateCollectionCleanupTest, DistributedMLTComponentTest, TestIndexSearcher, SOLR749Test, TestExactStatsCache, TestCSVLoader, HttpPartitionTest, TestSchemaNameResource, TestSchemaSimilarityResource, ConvertedLegacyTest, CollectionReloadTest, LargeFieldTest, TestSubQueryTransformer, TestAddFieldRealTimeGet, TestDynamicLoading, SchemaVersionSpecificBehaviorTest, CollectionsAPIDistributedZkTest, TestEmbeddedSolrServerConstructors, DistributedVersionInfoTest, TestPayloadCheckQParserPlugin, TestSearchPerf, ForceLeaderTest, RemoteQueryErrorTest, TestRandomFaceting, TestCollectionsAPIViaSolrCloudCluster, PeerSyncWithIndexFingerprintCachingTest, TestLuceneMatchVersion, TestDistributedMissingSort, TestFieldCache, TestSmileRequest, SpellCheckCollatorWithCollapseTest, CoreSorterTest, InfixSuggestersTest, TestPullReplica, BadIndexSchemaTest, TestSolrCloudWithDelegationTokens, TestSolrQueryResponse, SolrCoreCheckLockOnStartupTest, TestRebalanceLeaders, TestTrie, ReplicationFactorTest, TestImpersonationWithHadoopAuth, TestBlendedInfixSuggestions, TestCloudSchemaless, TestQueryWrapperFilter, OverriddenZkACLAndCredentialsProvidersTest, SolrJmxReporterTest, TestComponentsName, BJQParserTest, BasicZkTest, NodeMutatorTest, DistributedSuggestComponentTest, BooleanFieldTest, ExplicitHLLTest, TestMaxTokenLenTokenizer, TestPushWriter, TestOnReconnectListenerSupport, TestOrdValues, TestBinaryResponseWriter, CdcrReplicationHandlerTest, ManagedSchemaRoundRobinCloudTest, UpdateParamsTest, TestCloudPivotFacet, DistributedIntervalFacetingTest, TestSubQueryTransformerDistrib, PropertiesRequestHandlerTest, TestDynamicFieldCollectionResource, AssignBackwardCompatibilityTest, TestSuggestSpellingConverter, TestRemoteStreaming, ShardRoutingCustomTest, V2StandaloneTest, MBeansHandlerTest, TestClassicSimilarityFactory, ChaosMonkeySafeLeaderWithPullReplicasTest, JsonLoaderTest, SuggesterFSTTest, PolyFieldTest, DeleteInactiveReplicaTest, TestNumericRangeQuery64, TestExportWriter, TriLevelCompositeIdRoutingTest, HdfsTlogReplayBufferedWhileIndexingTest, TestCloudRecovery, HighlighterMaxOffsetTest, DirectSolrSpellCheckerTest, DistributedFacetPivotSmallTest, AnalyticsQueryTest, TestQueryTypes, PKIAuthenticationIntegrationTest, PingRequestHandlerTest, DistributedQueryComponentCustomSortTest, OpenExchangeRatesOrgProviderTest, HdfsRecoveryZkTest, TestApiFramework, FullSolrCloudDistribCmdsTest, ZkNodePropsTest, ParsingFieldUpdateProcessorsTest, UUIDUpdateProcessorFallbackTest, CloudMLTQParserTest, SearchHandlerTest, SparseHLLTest, CdcrBootstrapTest, DocValuesMissingTest, TestExclusionRuleCollectionAccess, DistribCursorPagingTest, TermVectorComponentDistributedTest, TestClusterStateMutator, TestDistribDocBasedVersion, BufferStoreTest, PeerSyncReplicationTest, TestCrossCoreJoin, StatsComponentTest, DistributedTermsComponentTest, TestBulkSchemaAPI, ClassificationUpdateProcessorTest, TestSSLRandomization, TestManagedSchemaAPI, InfoHandlerTest, SignatureUpdateProcessorFactoryTest, TestPolicyCloud, TestLuceneIndexBackCompat, SolrCloudExampleTest, TestComplexPhraseLeadingWildcard, TestStressReorder, HardAutoCommitTest, SimpleFacetsTest, SolrCoreTest, PeerSyncTest, DirectUpdateHandlerTest, HighlighterTest, SoftAutoCommitTest, TestIndexingPerformance, FieldAnalysisRequestHandlerTest, FastVectorHighlighterTest, IndexSchemaRuntimeFieldTest, TestCollationField, TestCSVResponseWriter, TestAnalyzedSuggestions, TestLFUCache, TestSolrJ, TestLRUCache, TestDocumentBuilder, SystemInfoHandlerTest, SolrTestCaseJ4Test, TestJettySolrRunner, ConnectionReuseTest, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrVersionReplicationTest, CleanupOldIndexTest, CollectionStateFormat2Test, CollectionTooManyReplicasTest, ConcurrentDeleteAndCreateCollectionTest, ConfigSetsAPITest, ConnectionManagerTest, DeleteNodeTest, DeleteReplicaTest, DeleteStatusTest, DistributedQueueTest, MigrateRouteKeyTest, MoveReplicaHDFSUlogDirTest, OverseerStatusTest, RestartWhileUpdatingTest, RollingRestartTest, SSLMigrationTest, SaslZkACLProviderTest, SharedFSAutoReplicaFailoverTest, SolrCLIZkUtilsTest, SolrXmlInZkTest, TestAuthenticationFramework, TestCloudInspectUtil, TestCloudJSONFacetJoinDomain, TestCloudSearcherWarming, TestClusterProperties, TestCollectionAPI, TestConfigSetsAPIExclusivity, TestCryptoKeys, TestDeleteCollectionOnDownNodes, TestDownShardTolerantSearch, TestLeaderElectionWithEmptyReplica, TestLeaderInitiatedRecoveryThread, TestLockTree, TestPrepRecovery, TestSegmentSorting, TestSolrCloudWithKerberosAlt, TestSolrCloudWithSecureImpersonation, TestStressCloudBlindAtomicUpdates, TestTlogReplica]
   [junit4] Completed [553/734 (1!)] on J1 in 123.08s, 13 tests, 1 failure <<< FAILURES!

[...truncated 47441 lines...]

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1456 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1456/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC

3 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.SystemLogListenerTest

Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:64210 within 30000 ms

Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:64210 within 30000 ms
	at __randomizedtesting.SeedInfo.seed([C11663E6AE8AF65E]:0)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:183)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:232)
	at org.apache.solr.cloud.SolrCloudTestCase$Builder.configure(SolrCloudTestCase.java:190)
	at org.apache.solr.cloud.autoscaling.SystemLogListenerTest.setupCluster(SystemLogListenerTest.java:82)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:64210 within 30000 ms
	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:233)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:175)
	... 30 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.SystemLogListenerTest

Error Message:
4 threads leaked from SUITE scope at org.apache.solr.cloud.autoscaling.SystemLogListenerTest:     1) Thread[id=35688, name=SessionTracker, state=TIMED_WAITING, group=TGRP-SystemLogListenerTest]         at java.lang.Object.wait(Native Method)         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)    2) Thread[id=35686, name=Thread-9801, state=WAITING, group=TGRP-SystemLogListenerTest]         at java.lang.Object.wait(Native Method)         at java.lang.Thread.join(Thread.java:1252)         at java.lang.Thread.join(Thread.java:1326)         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)    3) Thread[id=35690, name=ProcessThread(sid:0 cport:64210):, state=WAITING, group=TGRP-SystemLogListenerTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)    4) Thread[id=35687, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-SystemLogListenerTest]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.autoscaling.SystemLogListenerTest: 
   1) Thread[id=35688, name=SessionTracker, state=TIMED_WAITING, group=TGRP-SystemLogListenerTest]
        at java.lang.Object.wait(Native Method)
        at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
   2) Thread[id=35686, name=Thread-9801, state=WAITING, group=TGRP-SystemLogListenerTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Thread.join(Thread.java:1252)
        at java.lang.Thread.join(Thread.java:1326)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)
        at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
        at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   3) Thread[id=35690, name=ProcessThread(sid:0 cport:64210):, state=WAITING, group=TGRP-SystemLogListenerTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)
   4) Thread[id=35687, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-SystemLogListenerTest]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([C11663E6AE8AF65E]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.SystemLogListenerTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=35687, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-SystemLogListenerTest]         at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)         at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)         at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=35687, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-SystemLogListenerTest]
        at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)
        at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)
        at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
        at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([C11663E6AE8AF65E]:0)




Build Log:
[...truncated 13224 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.SystemLogListenerTest
   [junit4]   2> 4011006 INFO  (SUITE-SystemLogListenerTest-seed#[C11663E6AE8AF65E]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SystemLogListenerTest_C11663E6AE8AF65E-001/init-core-data-001
   [junit4]   2> 4011007 INFO  (SUITE-SystemLogListenerTest-seed#[C11663E6AE8AF65E]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 4011009 INFO  (SUITE-SystemLogListenerTest-seed#[C11663E6AE8AF65E]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 4011010 INFO  (SUITE-SystemLogListenerTest-seed#[C11663E6AE8AF65E]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.SystemLogListenerTest_C11663E6AE8AF65E-001/tempDir-001
   [junit4]   2> 4011010 INFO  (SUITE-SystemLogListenerTest-seed#[C11663E6AE8AF65E]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 4011011 INFO  (Thread-9801) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 4011011 INFO  (Thread-9801) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 4011017 ERROR (Thread-9801) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 4011111 INFO  (SUITE-SystemLogListenerTest-seed#[C11663E6AE8AF65E]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:64210
   [junit4]   2> 4041817 INFO  (SUITE-SystemLogListenerTest-seed#[C11663E6AE8AF65E]-worker) [    ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> Oct 05, 2017 7:38:13 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 4 leaked thread(s).
   [junit4]   2> Oct 05, 2017 7:38:33 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 4 threads leaked from SUITE scope at org.apache.solr.cloud.autoscaling.SystemLogListenerTest: 
   [junit4]   2>    1) Thread[id=35688, name=SessionTracker, state=TIMED_WAITING, group=TGRP-SystemLogListenerTest]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
   [junit4]   2>    2) Thread[id=35686, name=Thread-9801, state=WAITING, group=TGRP-SystemLogListenerTest]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)
   [junit4]   2>         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
   [junit4]   2>         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   [junit4]   2>    3) Thread[id=35690, name=ProcessThread(sid:0 cport:64210):, state=WAITING, group=TGRP-SystemLogListenerTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)
   [junit4]   2>    4) Thread[id=35687, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-SystemLogListenerTest]
   [junit4]   2>         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
   [junit4]   2>         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
   [junit4]   2>         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
   [junit4]   2>         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]   2>         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]   2>         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Oct 05, 2017 7:38:33 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=35688, name=SessionTracker, state=TIMED_WAITING, group=TGRP-SystemLogListenerTest]
   [junit4]   2>    2) Thread[id=35686, name=Thread-9801, state=WAITING, group=TGRP-SystemLogListenerTest]
   [junit4]   2>    3) Thread[id=35690, name=ProcessThread(sid:0 cport:64210):, state=WAITING, group=TGRP-SystemLogListenerTest]
   [junit4]   2>    4) Thread[id=35687, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-SystemLogListenerTest]
   [junit4]   2> 4062115 ERROR (ProcessThread(sid:0 cport:64210):) [    ] o.a.z.s.ZooKeeperCriticalThread Severe unrecoverable error, from thread : ProcessThread(sid:0 cport:64210):
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2> 	at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)
   [junit4]   2> 4062115 WARN  (Thread-9801) [    ] o.a.s.c.ZkTestServer Server interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.lang.Object.wait(Native Method)
   [junit4]   2> 	at java.lang.Thread.join(Thread.java:1252)
   [junit4]   2> 	at java.lang.Thread.join(Thread.java:1326)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)
   [junit4]   2> 	at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
   [junit4]   2> 	at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   [junit4]   2> 4062118 ERROR (ProcessThread(sid:0 cport:64210):) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 4062115 ERROR (SessionTracker) [    ] o.a.z.s.ZooKeeperCriticalThread Severe unrecoverable error, from thread : SessionTracker
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.lang.Object.wait(Native Method)
   [junit4]   2> 	at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
   [junit4]   2> 4062118 ERROR (SessionTracker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> Oct 05, 2017 7:38:36 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=35687, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-SystemLogListenerTest]
   [junit4]   2>         at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)
   [junit4]   2>         at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)
   [junit4]   2>         at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)
   [junit4]   2>         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
   [junit4]   2>         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
   [junit4]   2>         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)
   [junit4]   2>         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]   2>         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]   2>         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=true): {}, locale=ca-ES, timezone=Pacific/Guadalcanal
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=3,free=262362232,total=526385152
   [junit4]   2> NOTE: All tests run in this JVM: [TestReversedWildcardFilterFactory, CollectionStateFormat2Test, TestDocBasedVersionConstraints, TestSolrQueryResponse, TestMultiValuedNumericRangeQuery, TestFuzzyAnalyzedSuggestions, HdfsThreadLeakTest, OverseerCollectionConfigSetProcessorTest, MinimalSchemaTest, TestPullReplica, TestXmlQParserPlugin, MultiTermTest, AnalyticsQueryTest, SoftAutoCommitTest, TestIntervalFaceting, AssignTest, TestUpdate, OverseerRolesTest, LeaderInitiatedRecoveryOnShardRestartTest, UnloadDistributedZkTest, TestPseudoReturnFields, TestExclusionRuleCollectionAccess, ShufflingReplicaListTransformerTest, TestCustomStream, DistributedDebugComponentTest, SolrInfoBeanTest, ConfigureRecoveryStrategyTest, SolrMetricManagerTest, TestReplicationHandler, TestWriterPerf, TestManagedResourceStorage, CoreAdminRequestStatusTest, TestReloadAndDeleteDocs, DistribDocExpirationUpdateProcessorTest, SimplePostToolTest, SubstringBytesRefFilterTest, UUIDUpdateProcessorFallbackTest, CdcrReplicationDistributedZkTest, DeleteReplicaTest, AutoAddReplicasIntegrationTest, SolrCoreTest, TestQuerySenderNoQuery, TestPKIAuthenticationPlugin, MissingSegmentRecoveryTest, DirectUpdateHandlerTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, RulesTest, TestQuerySenderListener, TestBinaryField, TestFieldTypeCollectionResource, ConnectionReuseTest, HighlighterConfigTest, SortByFunctionTest, TestSolrCoreProperties, TestStressLiveNodes, BooleanFieldTest, QueryElevationComponentTest, OverseerTaskQueueTest, AlternateDirectoryTest, SuggestComponentContextFilterQueryTest, TestXIncludeConfig, InfoHandlerTest, AnalysisAfterCoreReloadTest, TestSolrXml, IndexSchemaTest, JSONWriterTest, TestGraphMLResponseWriter, CircularListTest, BaseCdcrDistributedZkTest, SolrJmxReporterTest, DirectUpdateHandlerOptimizeTest, TestStressVersions, DirectoryFactoryTest, TestSolrDeletionPolicy2, BlockCacheTest, DistributedQueryComponentCustomSortTest, SearchHandlerTest, TestImpersonationWithHadoopAuth, TestFieldCacheSort, TestOrdValues, TestRestoreCore, ZkControllerTest, BinaryUpdateRequestHandlerTest, TestEmbeddedSolrServerAdminHandler, TestReplicationHandlerBackup, DirectSolrConnectionTest, TestShortCircuitedRequests, DeleteLastCustomShardedReplicaTest, BigEndianAscendingWordSerializerTest, ExplicitHLLTest, TestDistributedStatsComponentCardinality, TestLockTree, ShardSplitTest, TestLeaderInitiatedRecoveryThread, SolrCloudExampleTest, FastVectorHighlighterTest, StressHdfsTest, TestSkipOverseerOperations, ReplaceNodeTest, DistributedFacetExistsSmallTest, TlogReplayBufferedWhileIndexingTest, DefaultValueUpdateProcessorTest, TestHighlightDedupGrouping, NotRequiredUniqueKeyTest, TestPHPSerializedResponseWriter, TestNamedUpdateProcessors, TestSort, TestPointFields, TestNonDefinedSimilarityFactory, CoreAdminHandlerTest, TestHdfsUpdateLog, TestBackupRepositoryFactory, TestFieldCollectionResource, TestRandomFlRTGCloud, TestIndexingPerformance, TestCoreContainer, ResponseHeaderTest, TestApiFramework, DistributedSpellCheckComponentTest, BasicDistributedZkTest, TestAddFieldRealTimeGet, LeaderFailureAfterFreshStartTest, CloudExitableDirectoryReaderTest, DistributedFacetPivotSmallTest, TestQueryWrapperFilter, LeaderElectionContextKeyTest, AddBlockUpdateTest, ConcurrentDeleteAndCreateCollectionTest, TestStressRecovery, ChaosMonkeyNothingIsSafeTest, XsltUpdateRequestHandlerTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestCloudPivotFacet, TestCloudRecovery, HdfsRecoverLeaseTest, TestPhraseSuggestions, FieldAnalysisRequestHandlerTest, TestTolerantUpdateProcessorCloud, TestRawTransformer, TestDistributedSearch, OutputWriterTest, TestExactStatsCache, RecoveryAfterSoftCommitTest, RegexBoostProcessorTest, RestartWhileUpdatingTest, TestHashQParserPlugin, DistanceFunctionTest, SSLMigrationTest, MoveReplicaHDFSTest, TestExtendedDismaxParser, TestRecoveryHdfs, TestFreeTextSuggestions, ReturnFieldsTest, NoCacheHeaderTest, SuggesterTest, TestLeaderElectionWithEmptyReplica, TestFieldCacheVsDocValues, ExecutePlanActionTest, LargeFieldTest, TestConfigReload, TestLegacyField, TestSearcherReuse, TestTestInjection, VMParamsZkACLAndCredentialsProvidersTest, RAMDirectoryFactoryTest, PrimitiveFieldTypeTest, TestFoldingMultitermQuery, HdfsRestartWhileUpdatingTest, QueryParsingTest, ClusterStateTest, TestSha256AuthenticationProvider, DisMaxRequestHandlerTest, TestDistribIDF, TestSurroundQueryParser, BasicAuthIntegrationTest, UniqFieldsUpdateProcessorFactoryTest, TestZkAclsWithHadoopAuth, ParsingFieldUpdateProcessorsTest, TestInPlaceUpdatesStandalone, TestRandomDVFaceting, ZkCLITest, TestFaceting, TestRecovery, TestRealTimeGet, TestStressReorder, TestJoin, DistributedTermsComponentTest, TestSolr4Spatial, SpellCheckComponentTest, TestGroupingSearch, PeerSyncTest, TestBadConfig, ShowFileRequestHandlerTest, SignatureUpdateProcessorFactoryTest, SuggesterFSTTest, DocValuesTest, DirectSolrSpellCheckerTest, TestOmitPositions, TermsComponentTest, TermVectorComponentTest, RequiredFieldsTest, SolrPluginUtilsTest, UpdateRequestProcessorFactoryTest, TestAnalyzedSuggestions, TestPartialUpdateDeduplication, TestLFUCache, TestLRUCache, TestDocumentBuilder, SliceStateTest, FileUtilsTest, CursorPagingTest, TestTolerantSearch, TestEmbeddedSolrServerConstructors, TestEmbeddedSolrServerSchemaAPI, AliasIntegrationTest, CdcrRequestHandlerTest, ChaosMonkeySafeLeaderWithPullReplicasTest, CleanupOldIndexTest, ConnectionManagerTest, CreateCollectionCleanupTest, CustomCollectionTest, DeleteShardTest, DistribCursorPagingTest, DocValuesNotIndexedTest, HttpPartitionTest, MoveReplicaTest, NodeMutatorTest, RemoteQueryErrorTest, ReplicationFactorTest, RollingRestartTest, SimpleCollectionCreateDeleteTest, TestCloudInspectUtil, TestCloudJSONFacetJoinDomain, TestClusterProperties, TestCollectionsAPIViaSolrCloudCluster, TestConfigSetsAPIExclusivity, TestCryptoKeys, TestLocalFSCloudBackupRestore, TestOnReconnectListenerSupport, TestReplicaProperties, TestRequestForwarding, TestRequestStatusCollectionAPI, TestSolrCloudWithKerberosAlt, TestSolrCloudWithSecureImpersonation, TestStressInPlaceUpdates, TestTolerantUpdateProcessorRandomCloud, ZkFailoverTest, HttpTriggerListenerTest, NodeLostTriggerTest, SystemLogListenerTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SystemLogListenerTest -Dtests.seed=C11663E6AE8AF65E -Dtests.slow=true -Dtests.locale=ca-ES -Dtests.timezone=Pacific/Guadalcanal -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | SystemLogListenerTest (suite) <<<
   [junit4]    > Throwable #1: org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:64210 within 30000 ms
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C11663E6AE8AF65E]:0)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:183)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
   [junit4]    > 	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:232)
   [junit4]    > 	at org.apache.solr.cloud.SolrCloudTestCase$Builder.configure(SolrCloudTestCase.java:190)
   [junit4]    > 	at org.apache.solr.cloud.autoscaling.SystemLogListenerTest.setupCluster(SystemLogListenerTest.java:82)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:64210 within 30000 ms
   [junit4]    > 	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:233)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:175)
   [junit4]    > 	... 30 moreThrowable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.autoscaling.SystemLogListenerTest: 
   [junit4]    >    1) Thread[id=35688, name=SessionTracker, state=TIMED_WAITING, group=TGRP-SystemLogListenerTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
   [junit4]    >    2) Thread[id=35686, name=Thread-9801, state=WAITING, group=TGRP-SystemLogListenerTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.lang.Thread.join(Thread.java:1252)
   [junit4]    >         at java.lang.Thread.join(Thread.java:1326)
   [junit4]    >         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)
   [junit4]    >         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
   [junit4]    >         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   [junit4]    >    3) Thread[id=35690, name=ProcessThread(sid:0 cport:64210):, state=WAITING, group=TGRP-SystemLogListenerTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)
   [junit4]    >    4) Thread[id=35687, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-SystemLogListenerTest]
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C11663E6AE8AF65E]:0)Throwable #3: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=35687, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-SystemLogListenerTest]
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C11663E6AE8AF65E]:0)
   [junit4] Completed [564/745 (1!)] on J0 in 54.14s, 0 tests, 3 errors <<< FAILURES!

[...truncated 47455 lines...]
BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:826: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:706: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:693: Source checkout is dirty (unversioned/missing files) after running tests!!! Offending files:
* lucene/licenses/morfologik-ukrainian-search-3.7.5.jar.sha1

Total time: 96 minutes 7 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1455 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1455/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth:     1) Thread[id=14710, name=jetty-launcher-2838-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)    2) Thread[id=14715, name=jetty-launcher-2838-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth: 
   1) Thread[id=14710, name=jetty-launcher-2838-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   2) Thread[id=14715, name=jetty-launcher-2838-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
	at __randomizedtesting.SeedInfo.seed([9A88CB42154A7069]:0)




Build Log:
[...truncated 12247 lines...]
   [junit4] Suite: org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth
   [junit4]   2> 1291131 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[9A88CB42154A7069]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_9A88CB42154A7069-001/init-core-data-001
   [junit4]   2> 1291131 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[9A88CB42154A7069]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1291132 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[9A88CB42154A7069]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1291165 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[9A88CB42154A7069]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_9A88CB42154A7069-001/tempDir-001
   [junit4]   2> 1291165 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[9A88CB42154A7069]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1291165 INFO  (Thread-3219) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1291165 INFO  (Thread-3219) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1291165 ERROR (Thread-3219) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1291265 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[9A88CB42154A7069]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:48693
   [junit4]   2> 1291293 INFO  (jetty-launcher-2838-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1291293 INFO  (jetty-launcher-2838-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1291296 INFO  (jetty-launcher-2838-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@438368ab{/solr,null,AVAILABLE}
   [junit4]   2> 1291297 INFO  (jetty-launcher-2838-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@36bbe5e0{/solr,null,AVAILABLE}
   [junit4]   2> 1291297 INFO  (jetty-launcher-2838-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3f8642{HTTP/1.1,[http/1.1]}{127.0.0.1:49542}
   [junit4]   2> 1291297 INFO  (jetty-launcher-2838-thread-1) [    ] o.e.j.s.Server Started @1293188ms
   [junit4]   2> 1291297 INFO  (jetty-launcher-2838-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=49542}
   [junit4]   2> 1291298 INFO  (jetty-launcher-2838-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@41e4895d{HTTP/1.1,[http/1.1]}{127.0.0.1:48877}
   [junit4]   2> 1291298 INFO  (jetty-launcher-2838-thread-2) [    ] o.e.j.s.Server Started @1293188ms
   [junit4]   2> 1291298 INFO  (jetty-launcher-2838-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=48877}
   [junit4]   2> 1291298 ERROR (jetty-launcher-2838-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1291298 ERROR (jetty-launcher-2838-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1291298 INFO  (jetty-launcher-2838-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1291298 INFO  (jetty-launcher-2838-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1291298 INFO  (jetty-launcher-2838-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1291298 INFO  (jetty-launcher-2838-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1291298 INFO  (jetty-launcher-2838-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1291298 INFO  (jetty-launcher-2838-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1291298 INFO  (jetty-launcher-2838-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-05T08:29:40.581Z
   [junit4]   2> 1291298 INFO  (jetty-launcher-2838-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-05T08:29:40.581Z
   [junit4]   2> 1291306 INFO  (jetty-launcher-2838-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1291322 INFO  (jetty-launcher-2838-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1291325 INFO  (jetty-launcher-2838-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48693/solr
   [junit4]   2> 1291330 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15eeba78ee00003, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1291333 INFO  (jetty-launcher-2838-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48693/solr
   [junit4]   2> 1291482 INFO  (jetty-launcher-2838-thread-1) [n:127.0.0.1:49542_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1291484 INFO  (jetty-launcher-2838-thread-2) [n:127.0.0.1:48877_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1291485 INFO  (jetty-launcher-2838-thread-2) [n:127.0.0.1:48877_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:48877_solr
   [junit4]   2> 1291485 INFO  (jetty-launcher-2838-thread-1) [n:127.0.0.1:49542_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49542_solr
   [junit4]   2> 1291486 INFO  (jetty-launcher-2838-thread-2) [n:127.0.0.1:48877_solr    ] o.a.s.c.Overseer Overseer (id=98775346737840134-127.0.0.1:48877_solr-n_0000000000) starting
   [junit4]   2> 1291487 INFO  (zkCallback-2846-thread-1-processing-n:127.0.0.1:49542_solr) [n:127.0.0.1:49542_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1291487 INFO  (zkCallback-2850-thread-1-processing-n:127.0.0.1:48877_solr) [n:127.0.0.1:48877_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1291496 INFO  (jetty-launcher-2838-thread-2) [n:127.0.0.1:48877_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48877_solr
   [junit4]   2> 1291497 INFO  (zkCallback-2846-thread-1-processing-n:127.0.0.1:49542_solr) [n:127.0.0.1:49542_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1291498 INFO  (zkCallback-2850-thread-1-processing-n:127.0.0.1:48877_solr) [n:127.0.0.1:48877_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1291548 INFO  (jetty-launcher-2838-thread-1) [n:127.0.0.1:49542_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HadoopAuthPlugin
   [junit4]   2> 1291551 INFO  (jetty-launcher-2838-thread-1) [n:127.0.0.1:49542_solr    ] o.a.s.s.HadoopAuthPlugin Params: {proxyuser.anyHostUsersGroup.hosts=*, proxyuser.wrongHost.hosts=1.1.1.1.1.1, proxyuser.bogusGroup.groups=__some_bogus_group, proxyuser.localHostAnyGroup.groups=*, cookie.domain=127.0.0.1, proxyuser.wrongHost.groups=*, type=simple, proxyuser.noHosts.groups=*, proxyuser.anyHostUsersGroup.groups=staff, proxyuser.noGroups.hosts=*, proxyuser.anyHostAnyUser.groups=*, delegation-token.token-kind=solr-dt, proxyuser.bogusGroup.hosts=*, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, signer.secret.provider.zookeeper.path=/token, zk-dt-secret-manager.enable=true, proxyuser.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, proxyuser.anyHostAnyUser.hosts=*, signer.secret.provider=zookeeper}
   [junit4]   2> 1291555 INFO  (jetty-launcher-2838-thread-1) [n:127.0.0.1:49542_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 1291572 INFO  (jetty-launcher-2838-thread-1-EventThread) [n:127.0.0.1:49542_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 1291573 INFO  (jetty-launcher-2838-thread-2) [n:127.0.0.1:48877_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HadoopAuthPlugin
   [junit4]   2> 1291574 INFO  (jetty-launcher-2838-thread-2) [n:127.0.0.1:48877_solr    ] o.a.s.s.HadoopAuthPlugin Params: {proxyuser.anyHostUsersGroup.hosts=*, proxyuser.wrongHost.hosts=1.1.1.1.1.1, proxyuser.bogusGroup.groups=__some_bogus_group, proxyuser.localHostAnyGroup.groups=*, cookie.domain=127.0.0.1, proxyuser.wrongHost.groups=*, type=simple, proxyuser.noHosts.groups=*, proxyuser.anyHostUsersGroup.groups=staff, proxyuser.noGroups.hosts=*, proxyuser.anyHostAnyUser.groups=*, delegation-token.token-kind=solr-dt, proxyuser.bogusGroup.hosts=*, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, signer.secret.provider.zookeeper.path=/token, zk-dt-secret-manager.enable=true, proxyuser.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, proxyuser.anyHostAnyUser.hosts=*, signer.secret.provider=zookeeper}
   [junit4]   2> 1291574 INFO  (jetty-launcher-2838-thread-2) [n:127.0.0.1:48877_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 1291578 INFO  (jetty-launcher-2838-thread-2-EventThread) [n:127.0.0.1:48877_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 1291597 INFO  (jetty-launcher-2838-thread-1) [n:127.0.0.1:49542_solr    ] o.a.s.c.CoreContainer PKIAuthenticationPlugin is managing internode requests
   [junit4]   2> 1291614 INFO  (jetty-launcher-2838-thread-2) [n:127.0.0.1:48877_solr    ] o.a.s.c.CoreContainer PKIAuthenticationPlugin is managing internode requests
   [junit4]   2> 1291656 INFO  (jetty-launcher-2838-thread-2) [n:127.0.0.1:48877_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48877.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@67fe6531
   [junit4]   2> 1291657 INFO  (jetty-launcher-2838-thread-1) [n:127.0.0.1:49542_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49542.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@67fe6531
   [junit4]   2> 1291662 INFO  (jetty-launcher-2838-thread-2) [n:127.0.0.1:48877_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48877.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@67fe6531
   [junit4]   2> 1291662 INFO  (jetty-launcher-2838-thread-2) [n:127.0.0.1:48877_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48877.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@67fe6531
   [junit4]   2> 1291663 INFO  (jetty-launcher-2838-thread-1) [n:127.0.0.1:49542_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49542.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@67fe6531
   [junit4]   2> 1291663 INFO  (jetty-launcher-2838-thread-1) [n:127.0.0.1:49542_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49542.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@67fe6531
   [junit4]   2> 1291663 INFO  (jetty-launcher-2838-thread-2) [n:127.0.0.1:48877_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_9A88CB42154A7069-001/tempDir-001/node2/.
   [junit4]   2> 1291663 INFO  (jetty-launcher-2838-thread-1) [n:127.0.0.1:49542_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_9A88CB42154A7069-001/tempDir-001/node1/.
   [junit4]   2> 1291702 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[9A88CB42154A7069]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1291703 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[9A88CB42154A7069]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48693/solr ready
   [junit4]   2> 1291856 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigHosts-seed#[9A88CB42154A7069]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigHosts
   [junit4]   2> 1291877 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigHosts-seed#[9A88CB42154A7069]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigHosts
   [junit4]   2> 1292009 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateHost-seed#[9A88CB42154A7069]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateHost
   [junit4]   2> 1292014 INFO  (qtp776626350-14678) [n:127.0.0.1:48877_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&user.name=localHostAnyGroup&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1292014 INFO  (qtp776626350-14678) [n:127.0.0.1:48877_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&user.name=localHostAnyGroup&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1292014 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateHost-seed#[9A88CB42154A7069]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateHost
   [junit4]   2> 1292145 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigGroups-seed#[9A88CB42154A7069]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigGroups
   [junit4]   2> 1292160 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigGroups-seed#[9A88CB42154A7069]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigGroups
   [junit4]   2> 1292300 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateAnyHostAnyUser-seed#[9A88CB42154A7069]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateAnyHostAnyUser
   [junit4]   2> 1292302 INFO  (qtp776626350-14678) [n:127.0.0.1:48877_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&user.name=anyHostAnyUser&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1292302 INFO  (qtp776626350-14678) [n:127.0.0.1:48877_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&user.name=anyHostAnyUser&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1292302 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateAnyHostAnyUser-seed#[9A88CB42154A7069]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateAnyHostAnyUser
   [junit4]   2> 1292433 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidGroup-seed#[9A88CB42154A7069]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidGroup
   [junit4]   2> 1292457 WARN  (qtp776626350-14673) [    ] o.a.h.s.ShellBasedUnixGroupsMapping got exception trying to get groups for user bar: id: invalid user name: "bar"
   [junit4]   2> id: invalid user name: "bar"
   [junit4]   2> 
   [junit4]   2> 1292461 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidGroup-seed#[9A88CB42154A7069]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidGroup
   [junit4]   2> 1292612 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateGroup-seed#[9A88CB42154A7069]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateGroup
   [junit4]   2> 1292614 INFO  (qtp776626350-14676) [n:127.0.0.1:48877_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=jenkins&action=LIST&user.name=anyHostUsersGroup&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1292614 INFO  (qtp776626350-14676) [n:127.0.0.1:48877_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=jenkins&action=LIST&user.name=anyHostUsersGroup&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1292614 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateGroup-seed#[9A88CB42154A7069]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateGroup
   [junit4]   2> 1292750 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidProxyUser-seed#[9A88CB42154A7069]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidProxyUser
   [junit4]   2> 1292754 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidProxyUser-seed#[9A88CB42154A7069]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidProxyUser
   [junit4]   2> 1292890 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyWrongHost-seed#[9A88CB42154A7069]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyWrongHost
   [junit4]   2> 1307909 WARN  (jetty-launcher-2838-thread-1-SendThread(127.0.0.1:48693)) [n:127.0.0.1:49542_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 16290ms for sessionid 0x15eeba78ee00007
   [junit4]   2> 1307910 WARN  (jetty-launcher-2838-thread-1-SendThread(127.0.0.1:48693)) [n:127.0.0.1:49542_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 16245ms for sessionid 0x15eeba78ee00004
   [junit4]   2> 1307912 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyWrongHost-seed#[9A88CB42154A7069]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyWrongHost
   [junit4]   2> 1307913 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15eeba78ee00004, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1307913 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15eeba78ee00007, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1307910 WARN  (jetty-launcher-2838-thread-2-SendThread(127.0.0.1:48693)) [n:127.0.0.1:48877_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 16245ms for sessionid 0x15eeba78ee00006
   [junit4]   2> 1307909 WARN  (jetty-launcher-2838-thread-2-SendThread(127.0.0.1:48693)) [n:127.0.0.1:48877_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 16288ms for sessionid 0x15eeba78ee00008
   [junit4]   2> 1307913 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15eeba78ee00006, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1307920 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15eeba78ee00008, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1308011 WARN  (zkCallback-2856-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@739709c4 name: ZooKeeperConnection Watcher:127.0.0.1:48693/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1308012 WARN  (zkCallback-2856-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 1308014 INFO  (jetty-launcher-2838-thread-1-EventThread) [n:127.0.0.1:49542_solr    ] o.a.c.f.s.ConnectionStateManager State change: SUSPENDED
   [junit4]   2> 1308016 WARN  (zkCallback-2846-thread-3-processing-n:127.0.0.1:49542_solr) [n:127.0.0.1:49542_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3fadfaa6 name: ZooKeeperConnection Watcher:127.0.0.1:48693/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1308016 WARN  (zkCallback-2846-thread-3-processing-n:127.0.0.1:49542_solr) [n:127.0.0.1:49542_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 1308022 WARN  (zkCallback-2850-thread-1-processing-n:127.0.0.1:48877_solr) [n:127.0.0.1:48877_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@b0fd95c name: ZooKeeperConnection Watcher:127.0.0.1:48693/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1308022 WARN  (zkCallback-2850-thread-1-processing-n:127.0.0.1:48877_solr) [n:127.0.0.1:48877_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 1308030 INFO  (jetty-launcher-2838-thread-2-EventThread) [n:127.0.0.1:48877_solr    ] o.a.c.f.s.ConnectionStateManager State change: SUSPENDED
   [junit4]   2> 1308089 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyNullProxyUser-seed#[9A88CB42154A7069]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNullProxyUser
   [junit4]   2> 1308091 INFO  (qtp776626350-14665) [    ] o.a.s.s.SolrDispatchFilter Error authenticating
   [junit4]   2> java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler.authenticate(PseudoAuthenticationHandler.java:195)
   [junit4]   2> 	at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> 	at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
   [junit4]   2> 	at org.apache.solr.security.HadoopAuthFilter.doFilter(HadoopAuthFilter.java:110)
   [junit4]   2> 	at org.apache.solr.security.HadoopAuthPlugin.doAuthenticate(HadoopAuthPlugin.java:245)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1308092 WARN  (qtp776626350-14665) [    ] o.e.j.s.ServletHandler /solr/admin/collections
   [junit4]   2> org.apache.solr.common.SolrException: Error during request authentication, 
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:457)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler.authenticate(PseudoAuthenticationHandler.java:195)
   [junit4]   2> 	at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> 	at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
   [junit4]   2> 	at org.apache.solr.security.HadoopAuthFilter.doFilter(HadoopAuthFilter.java:110)
   [junit4]   2> 	at org.apache.solr.security.HadoopAuthPlugin.doAuthenticate(HadoopAuthPlugin.java:245)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
   [junit4]   2> 	... 26 more
   [junit4]   2> 1308096 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyNullProxyUser-seed#[9A88CB42154A7069]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNullProxyUser
   [junit4] IGNOR/A 0.00s J0 | TestImpersonationWithHadoopAuth.testForwarding
   [junit4]    > Assumption #1: 'awaitsfix' test group is disabled (@AwaitsFix(bugUrl=https://issues.apache.org/jira/browse/HADOOP-9893))
   [junit4]   2> 1308806 INFO  (jetty-closer-2839-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@3f8642{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1308806 INFO  (jetty-closer-2839-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@41e4895d{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1308809 INFO  (jetty-closer-2839-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1757477537
   [junit4]   2> 1308811 INFO  (jetty-closer-2839-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1002081792
   [junit4]   2> 1308811 INFO  (jetty-closer-2839-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 1308811 INFO  (jetty-closer-2839-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@14768568: rootName = solr_48877, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@d58eca2
   [junit4]   2> 1308811 INFO  (jetty-closer-2839-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 1308811 INFO  (jetty-closer-2839-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3e7445c: rootName = solr_49542, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@752a6817
   [junit4]   2> 1308822 INFO  (jetty-closer-2839-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1308822 INFO  (jetty-closer-2839-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1cb50f68: rootName = solr_48877, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@2853a7df
   [junit4]   2> 1308822 INFO  (jetty-closer-2839-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1308822 INFO  (jetty-closer-2839-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7a2bd70c: rootName = solr_49542, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@2853a7df
   [junit4]   2> 1308824 INFO  (jetty-closer-2839-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1308824 INFO  (jetty-closer-2839-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@51f0bb96: rootName = solr_48877, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@4ee2e926
   [junit4]   2> 1308825 INFO  (jetty-closer-2839-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1308825 INFO  (jetty-closer-2839-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@779e0b35: rootName = solr_49542, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@4ee2e926
   [junit4]   2> 1309247 INFO  (jetty-closer-2839-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1309247 INFO  (jetty-closer-2839-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1309248 INFO  (jetty-closer-2839-thread-1) [    ] o.a.s.c.Overseer Overseer (id=98775346737840134-127.0.0.1:48877_solr-n_0000000000) closing
   [junit4]   2> 1309248 INFO  (OverseerStateUpdate-98775346737840134-127.0.0.1:48877_solr-n_0000000000) [n:127.0.0.1:48877_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:48877_solr
   [junit4]   2> 1309248 INFO  (OverseerCollectionConfigSetProcessor-98775346737840134-127.0.0.1:48877_solr-n_0000000000) [n:127.0.0.1:48877_solr    ] o.a.s.c.OverseerTaskProcessor According to ZK I (id=98775346737840134-127.0.0.1:48877_solr-n_0000000000) am no longer a leader.
   [junit4]   2> 1309250 WARN  (OverseerAutoScalingTriggerThread-98775346737840134-127.0.0.1:48877_solr-n_0000000000) [n:127.0.0.1:48877_solr    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 1309255 WARN  (zkCallback-2850-thread-5-processing-n:127.0.0.1:48877_solr) [n:127.0.0.1:48877_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1309256 ERROR (Thread[Thread-3238,5,TGRP-TestImpersonationWithHadoopAuth]) [n:127.0.0.1:48877_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 1309705 INFO  (jetty-closer-2839-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@36bbe5e0{/solr,null,UNAVAILABLE}
   [junit4]   2> 1309833 INFO  (zkCallback-2846-thread-4-processing-n:127.0.0.1:49542_solr) [n:127.0.0.1:49542_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:49542_solr
   [junit4]   2> 1309833 INFO  (zkCallback-2846-thread-5-processing-n:127.0.0.1:49542_solr) [n:127.0.0.1:49542_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 1309834 INFO  (jetty-closer-2839-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1309834 INFO  (jetty-closer-2839-thread-2) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1309836 WARN  (zkCallback-2846-thread-5-processing-n:127.0.0.1:49542_solr) [n:127.0.0.1:49542_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1311338 ERROR (Thread[Thread-3236,5,TGRP-TestImpersonationWithHadoopAuth]) [n:127.0.0.1:49542_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 1311345 INFO  (jetty-closer-2839-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@438368ab{/solr,null,UNAVAILABLE}
   [junit4]   2> 1311348 ERROR (SUITE-TestImpersonationWithHadoopAuth-seed#[9A88CB42154A7069]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1311348 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[9A88CB42154A7069]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:48693 48693
   [junit4]   2> 1314684 INFO  (Thread-3219) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:48693 48693
   [junit4]   2> 1314690 WARN  (Thread-3219) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	3	/solr/collections
   [junit4]   2> 	3	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 
   [junit4]   2> 1314690 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[9A88CB42154A7069]-worker) [    ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> Oct 05, 2017 8:30:03 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> Oct 05, 2017 8:30:24 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth: 
   [junit4]   2>    1) Thread[id=14710, name=jetty-launcher-2838-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2>         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2>         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2>         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2>         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2>    2) Thread[id=14715, name=jetty-launcher-2838-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2>         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2>         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2>         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2>         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> Oct 05, 2017 8:30:24 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=14710, name=jetty-launcher-2838-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]   2>    2) Thread[id=14715, name=jetty-launcher-2838-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]   2> 1334818 ERROR (jetty-launcher-2838-thread-1-EventThread) [n:127.0.0.1:49542_solr    ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2> 	at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2> 	at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2> 	at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2> 	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> 1334818 ERROR (jetty-launcher-2838-thread-2-EventThread) [n:127.0.0.1:48877_solr    ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2> 	at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2> 	at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2> 	at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2> 	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> Oct 05, 2017 8:30:24 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=true): {}, locale=ar-LY, timezone=America/Argentina/Jujuy
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=125433136,total=388169728
   [junit4]   2> NOTE: All tests run in this JVM: [HdfsDirectoryFactoryTest, TestCustomStream, DistributedFacetPivotLongTailTest, TestPseudoReturnFields, TestSolrCloudWithKerberosAlt, TestStressVersions, TestQuerySenderListener, MissingSegmentRecoveryTest, MultiThreadedOCPTest, DistributedQueueTest, SimpleCollectionCreateDeleteTest, DateMathParserTest, SegmentsInfoRequestHandlerTest, TestSchemalessBufferedUpdates, TestGraphMLResponseWriter, ResourceLoaderTest, HardAutoCommitTest, TestSha256AuthenticationProvider, TestHdfsUpdateLog, CloudMLTQParserTest, BlockCacheTest, TestSolrQueryResponse, TestConfigSetImmutable, URLClassifyProcessorTest, ReplaceNodeTest, ZkCLITest, PeerSyncWithIndexFingerprintCachingTest, DistributedQueryComponentCustomSortTest, TestConfigReload, TestSmileRequest, DirectUpdateHandlerTest, TestFieldTypeCollectionResource, PeerSyncTest, TestBackupRepositoryFactory, SearchHandlerTest, CoreAdminHandlerTest, AutoScalingHandlerTest, TestLMDirichletSimilarityFactory, NoCacheHeaderTest, DistributedFacetPivotSmallTest, TimeZoneUtilsTest, TestCodecSupport, TestManagedSchemaAPI, AlternateDirectoryTest, TestInitParams, HdfsLockFactoryTest, GraphQueryTest, TestCSVResponseWriter, DisMaxRequestHandlerTest, CdcrRequestHandlerTest, TestLeaderElectionWithEmptyReplica, HealthCheckHandlerTest, TestSchemaNameResource, TestBinaryField, TestStressRecovery, ExecutePlanActionTest, ChaosMonkeyNothingIsSafeTest, CdcrReplicationDistributedZkTest, TestSolrCoreProperties, InfixSuggestersTest, MultiTermTest, TestAddFieldRealTimeGet, TestConfigSets, TestFieldCacheSort, LargeFieldTest, ConcurrentDeleteAndCreateCollectionTest, SimplePostToolTest, SubstringBytesRefFilterTest, ExplicitHLLTest, OutOfBoxZkACLAndCredentialsProvidersTest, AssignTest, TestSSLRandomization, TestFoldingMultitermQuery, UUIDFieldTest, TestIndexingPerformance, TestRandomDVFaceting, TestPHPSerializedResponseWriter, DistributedSpellCheckComponentTest, SpellCheckCollatorWithCollapseTest, TestNumericRangeQuery64, TestSolrCoreParser, TestCustomSort, LeaderFailureAfterFreshStartTest, ZkControllerTest, ResponseHeaderTest, TestMultiValuedNumericRangeQuery, TestReversedWildcardFilterFactory, TestFieldCollectionResource, SuggesterWFSTTest, TestCloudSchemaless, TestXIncludeConfig, TestBulkSchemaConcurrent, TestClusterProperties, TlogReplayBufferedWhileIndexingTest, SoftAutoCommitTest, MinimalSchemaTest, OverseerTest, TestFaceting, PathHierarchyTokenizerFactoryTest, SolrTestCaseJ4Test, ScriptEngineTest, DistributedFacetExistsSmallTest, ConnectionReuseTest, SolrCloudReportersTest, BasicAuthIntegrationTest, StressHdfsTest, TestConfigsApi, TestMacroExpander, ReturnFieldsTest, SSLMigrationTest, TestSkipOverseerOperations, ClusterStateTest, TestBlendedInfixSuggestions, SolrMetricsIntegrationTest, TestNamedUpdateProcessors, DocExpirationUpdateProcessorFactoryTest, RecoveryAfterSoftCommitTest, TestStressLiveNodes, TestIntervalFaceting, TestBulkSchemaAPI, TestPushWriter, TestNestedDocsSort, TestFieldCacheVsDocValues, ClassificationUpdateProcessorFactoryTest, SolrGraphiteReporterTest, NotRequiredUniqueKeyTest, TestLegacyNumericUtils, TestSimpleTextCodec, SolrIndexSplitterTest, TestPayloadCheckQParserPlugin, PKIAuthenticationIntegrationTest, FieldAnalysisRequestHandlerTest, SyncSliceTest, SolrCoreTest, TestBinaryResponseWriter, ClassificationUpdateProcessorTest, TestCloudInspectUtil, TestSolrCoreSnapshots, TestSubQueryTransformer, TestQuerySenderNoQuery, TestFunctionQuery, TestSQLHandlerNonCloud, WordBreakSolrSpellCheckerTest, WrapperMergePolicyFactoryTest, TestImpersonationWithHadoopAuth]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestImpersonationWithHadoopAuth -Dtests.seed=9A88CB42154A7069 -Dtests.slow=true -Dtests.locale=ar-LY -Dtests.timezone=America/Argentina/Jujuy -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J0 | TestImpersonationWithHadoopAuth (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth: 
   [junit4]    >    1) Thread[id=14710, name=jetty-launcher-2838-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]    >    2) Thread[id=14715, name=jetty-launcher-2838-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9A88CB42154A7069]:0)
   [junit4] Completed [263/745 (1!)] on J0 in 43.71s, 10 tests, 1 error, 1 skipped <<< FAILURES!

[...truncated 44597 lines...]
-ecj-javadoc-lint-src:
    [mkdir] Created dir: /var/tmp/ecj41246448
 [ecj-lint] Compiling 1149 source files to /var/tmp/ecj41246448
 [ecj-lint] invalid Class-Path header in manifest of jar file: /export/home/jenkins/.ivy2/cache/org.restlet.jee/org.restlet/jars/org.restlet-2.3.0.jar
 [ecj-lint] invalid Class-Path header in manifest of jar file: /export/home/jenkins/.ivy2/cache/org.restlet.jee/org.restlet.ext.servlet/jars/org.restlet.ext.servlet-2.3.0.jar
 [ecj-lint] ----------
 [ecj-lint] 1. ERROR in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java (at line 40)
 [ecj-lint] 	import java.util.concurrent.atomic.AtomicReference;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import java.util.concurrent.atomic.AtomicReference is never used
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 2. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/core/CoreContainer.java (at line 1033)
 [ecj-lint] 	core = new SolrCore(this, dcore, coreConfig);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'core' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 3. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java (at line 234)
 [ecj-lint] 	dir = new BlockDirectory(path, hdfsDir, cache, null, blockCacheReadEnabled, false, cacheMerges, cacheReadOnce);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'dir' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 4. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (at line 121)
 [ecj-lint] 	reader = cfiltfac.create(reader);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'reader' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] 5. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (at line 145)
 [ecj-lint] 	return namedList;
 [ecj-lint] 	^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'listBasedTokenStream' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 6. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java (at line 1285)
 [ecj-lint] 	DirectoryReader reader = s==null ? null : s.get().getIndexReader();
 [ecj-lint] 	                ^^^^^^
 [ecj-lint] Resource leak: 'reader' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 7. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java (at line 517)
 [ecj-lint] 	ParallelStream parallelStream = new ParallelStream(zk, collection, tupleStream, numWorkers, comp);
 [ecj-lint] 	               ^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'parallelStream' is never closed
 [ecj-lint] ----------
 [ecj-lint] 8. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java (at line 743)
 [ecj-lint] 	ParallelStream parallelStream = new ParallelStream(zkHost, collection, tupleStream, numWorkers, comp);
 [ecj-lint] 	               ^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'parallelStream' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 9. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java (at line 578)
 [ecj-lint] 	tvWindowStream = new OffsetWindowTokenFilter(tvStream);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'tvWindowStream' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 10. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (at line 925)
 [ecj-lint] 	fastForRandomSet = new HashDocSet(sset.getDocs(), 0, sset.size());
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'fastForRandomSet' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 11. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/response/SmileResponseWriter.java (at line 33)
 [ecj-lint] 	new SmileWriter(out, request, response).writeResponse();
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 12. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java (at line 146)
 [ecj-lint] 	ratesJsonStream = resourceLoader.openResource(ratesFileLocation);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'ratesJsonStream' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 13. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java (at line 1315)
 [ecj-lint] 	UninvertingReader uninvertingReader = new UninvertingReader(new ReaderWrapper(searcher.getSlowAtomicReader(), collapseField), mapping);
 [ecj-lint] 	                  ^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'uninvertingReader' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 14. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 145)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] 15. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 154)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] 16. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 163)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 17. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java (at line 79)
 [ecj-lint] 	SolrIndexSearcher is = (SolrIndexSearcher) o;
 [ecj-lint] 	                  ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 18. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java (at line 79)
 [ecj-lint] 	SolrIndexSearcher is = (SolrIndexSearcher) o;
 [ecj-lint] 	                  ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 19. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java (at line 291)
 [ecj-lint] 	throw new SolrException(ErrorCode.SERVER_ERROR, "Error occurred while loading solr.xml from zookeeper", e);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'loader' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 20. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/update/CdcrTransactionLog.java (at line 98)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 21. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (at line 191)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 22. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (at line 261)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 23. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 228)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] 24. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 228)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] 25. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 260)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 26. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 275)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 27. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 529)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 28. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (at line 655)
 [ecj-lint] 	BAOS bos =  new BAOS();
 [ecj-lint] 	     ^^^
 [ecj-lint] Resource leak: 'bos' is never closed
 [ecj-lint] ----------
 [ecj-lint] 29. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (at line 655)
 [ecj-lint] 	BAOS bos =  new BAOS();
 [ecj-lint] 	     ^^^
 [ecj-lint] Resource leak: 'bos' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 30. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/util/SolrLogLayout.java (at line 149)
 [ecj-lint] 	SolrQueryRequest req = requestInfo == null ? null : requestInfo.getReq();
 [ecj-lint] 	                 ^^^
 [ecj-lint] Resource leak: 'req' is never closed
 [ecj-lint] ----------
 [ecj-lint] 30 problems (1 error, 29 warnings)

BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:826: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:101: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build.xml:684: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/common-build.xml:2081: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/common-build.xml:2120: Compile failed; see the compiler error output for details.

Total time: 96 minutes 26 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1454 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1454/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC

6 tests failed.
FAILED:  org.apache.solr.cloud.CdcrBootstrapTest.testConvertClusterToCdcrAndBootstrap

Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:42976 within 30000 ms

Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:42976 within 30000 ms
	at __randomizedtesting.SeedInfo.seed([AE584CE27515D63E:798F6395C14A4E79]:0)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:183)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:232)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:195)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:122)
	at org.apache.solr.cloud.CdcrBootstrapTest.testConvertClusterToCdcrAndBootstrap(CdcrBootstrapTest.java:61)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:42976 within 30000 ms
	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:233)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:175)
	... 46 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CdcrBootstrapTest

Error Message:
4 threads leaked from SUITE scope at org.apache.solr.cloud.CdcrBootstrapTest:     1) Thread[id=27281, name=Thread-6704, state=WAITING, group=TGRP-CdcrBootstrapTest]         at java.lang.Object.wait(Native Method)         at java.lang.Thread.join(Thread.java:1252)         at java.lang.Thread.join(Thread.java:1326)         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)    2) Thread[id=27285, name=ProcessThread(sid:0 cport:42976):, state=WAITING, group=TGRP-CdcrBootstrapTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)    3) Thread[id=27282, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-CdcrBootstrapTest]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=27283, name=SessionTracker, state=TIMED_WAITING, group=TGRP-CdcrBootstrapTest]         at java.lang.Object.wait(Native Method)         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.CdcrBootstrapTest: 
   1) Thread[id=27281, name=Thread-6704, state=WAITING, group=TGRP-CdcrBootstrapTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Thread.join(Thread.java:1252)
        at java.lang.Thread.join(Thread.java:1326)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)
        at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
        at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   2) Thread[id=27285, name=ProcessThread(sid:0 cport:42976):, state=WAITING, group=TGRP-CdcrBootstrapTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)
   3) Thread[id=27282, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-CdcrBootstrapTest]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=27283, name=SessionTracker, state=TIMED_WAITING, group=TGRP-CdcrBootstrapTest]
        at java.lang.Object.wait(Native Method)
        at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
	at __randomizedtesting.SeedInfo.seed([AE584CE27515D63E]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CdcrBootstrapTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=27282, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-CdcrBootstrapTest]         at sun.nio.ch.IOUtil.drain(Native Method)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:109)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=27282, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-CdcrBootstrapTest]
        at sun.nio.ch.IOUtil.drain(Native Method)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:109)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([AE584CE27515D63E]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:     1) Thread[id=22317, name=jetty-launcher-4213-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)    2) Thread[id=22318, name=jetty-launcher-4213-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   1) Thread[id=22317, name=jetty-launcher-4213-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   2) Thread[id=22318, name=jetty-launcher-4213-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
	at __randomizedtesting.SeedInfo.seed([AE584CE27515D63E]:0)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication

Error Message:
Index: 0, Size: 0

Stack Trace:
java.lang.IndexOutOfBoundsException: Index: 0, Size: 0
	at __randomizedtesting.SeedInfo.seed([AE584CE27515D63E:BA1017B756126B20]:0)
	at java.util.ArrayList.rangeCheck(ArrayList.java:653)
	at java.util.ArrayList.get(ArrayList.java:429)
	at org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication(TestReplicationHandler.java:561)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry

Error Message:
Captured an uncaught exception in thread: Thread[id=18440, name=OverseerAutoScalingTriggerThread-98772829996646402-dummy.host.com:8984_solr-n_0000000000, state=RUNNABLE, group=Overseer autoscaling triggers]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=18440, name=OverseerAutoScalingTriggerThread-98772829996646402-dummy.host.com:8984_solr-n_0000000000, state=RUNNABLE, group=Overseer autoscaling triggers]
	at __randomizedtesting.SeedInfo.seed([AE584CE27515D63E:78876DB84308C5A6]:0)
Caused by: org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.auto_add_replicas
	at __randomizedtesting.SeedInfo.seed([AE584CE27515D63E]:0)
	at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:107)
	at org.apache.solr.cloud.autoscaling.TriggerEventQueue.<init>(TriggerEventQueue.java:44)
	at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.<init>(ScheduledTriggers.java:398)
	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.add(ScheduledTriggers.java:149)
	at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:220)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.auto_add_replicas
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
	at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:93)
	at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:78)
	at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:105)
	... 5 more




Build Log:
[...truncated 12129 lines...]
   [junit4] Suite: org.apache.solr.handler.TestReplicationHandler
   [junit4]   2> 1500520 INFO  (SUITE-TestReplicationHandler-seed#[AE584CE27515D63E]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/init-core-data-001
   [junit4]   2> 1500521 WARN  (SUITE-TestReplicationHandler-seed#[AE584CE27515D63E]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4 numCloses=4
   [junit4]   2> 1500523 INFO  (SUITE-TestReplicationHandler-seed#[AE584CE27515D63E]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 1500525 INFO  (SUITE-TestReplicationHandler-seed#[AE584CE27515D63E]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 1500542 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.SolrTestCaseJ4 ###Starting doTestReplicateAfterCoreReload
   [junit4]   2> 1500542 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/collection1
   [junit4]   2> 1500546 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1500547 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@a332755{/solr,null,AVAILABLE}
   [junit4]   2> 1500557 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@a6274a5{HTTP/1.1,[http/1.1]}{127.0.0.1:57561}
   [junit4]   2> 1500559 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.Server Started @1502772ms
   [junit4]   2> 1500559 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/collection1/data, hostContext=/solr, hostPort=57561}
   [junit4]   2> 1500559 ERROR (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1500560 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1500560 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 1500560 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1500560 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-04T21:48:09.482Z
   [junit4]   2> 1500560 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/solr.xml
   [junit4]   2> 1500563 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1500563 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1500564 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1500609 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd
   [junit4]   2> 1500615 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd
   [junit4]   2> 1500615 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd
   [junit4]   2> 1500617 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/.
   [junit4]   2> 1500617 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1500617 INFO  (coreLoadExecutor-2937-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1500633 INFO  (coreLoadExecutor-2937-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1500640 INFO  (coreLoadExecutor-2937-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1500642 INFO  (coreLoadExecutor-2937-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 1500642 INFO  (coreLoadExecutor-2937-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/./collection1, trusted=true
   [junit4]   2> 1500642 INFO  (coreLoadExecutor-2937-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd
   [junit4]   2> 1500642 INFO  (coreLoadExecutor-2937-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1500642 INFO  (coreLoadExecutor-2937-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/./collection1/data/]
   [junit4]   2> 1500643 INFO  (coreLoadExecutor-2937-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=2, maxMergeAtOnceExplicit=6, maxMergedSegmentMB=77.86328125, floorSegmentMB=0.9970703125, forceMergeDeletesPctAllowed=2.487897048783277, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.35354646909200416
   [junit4]   2> 1500699 INFO  (coreLoadExecutor-2937-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1500699 INFO  (coreLoadExecutor-2937-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1500700 INFO  (coreLoadExecutor-2937-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=74.1171875, floorSegmentMB=0.7744140625, forceMergeDeletesPctAllowed=4.108848394977814, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.35848579947479964
   [junit4]   2> 1500700 INFO  (coreLoadExecutor-2937-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@52354db4[collection1] main]
   [junit4]   2> 1500700 INFO  (coreLoadExecutor-2937-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/collection1/conf
   [junit4]   2> 1500701 INFO  (coreLoadExecutor-2937-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following config files: schema.xml,xslt/dummy.xsl
   [junit4]   2> 1500701 INFO  (coreLoadExecutor-2937-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1500701 INFO  (searcherExecutor-2938-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@52354db4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1500702 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-002/collection1
   [junit4]   2> 1500704 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1500706 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6cd8f708{/solr,null,AVAILABLE}
   [junit4]   2> 1500706 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4d2a2dbd{HTTP/1.1,[http/1.1]}{127.0.0.1:41938}
   [junit4]   2> 1500706 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.Server Started @1502919ms
   [junit4]   2> 1500706 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-002/collection1/data, hostContext=/solr, hostPort=41938}
   [junit4]   2> 1500706 ERROR (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1500709 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1500709 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 1500709 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1500709 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-04T21:48:09.631Z
   [junit4]   2> 1500709 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-002/solr.xml
   [junit4]   2> 1500711 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1500712 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1500712 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1500743 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd
   [junit4]   2> 1500749 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd
   [junit4]   2> 1500750 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd
   [junit4]   2> 1500751 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-002/.
   [junit4]   2> 1500751 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1500752 INFO  (coreLoadExecutor-2947-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1500765 INFO  (coreLoadExecutor-2947-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1500770 INFO  (coreLoadExecutor-2947-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1500772 INFO  (coreLoadExecutor-2947-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 1500772 INFO  (coreLoadExecutor-2947-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-002/./collection1, trusted=true
   [junit4]   2> 1500772 INFO  (coreLoadExecutor-2947-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd
   [junit4]   2> 1500772 INFO  (coreLoadExecutor-2947-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1500772 INFO  (coreLoadExecutor-2947-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-002/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-002/./collection1/data/]
   [junit4]   2> 1500773 INFO  (coreLoadExecutor-2947-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=2, maxMergeAtOnceExplicit=6, maxMergedSegmentMB=77.86328125, floorSegmentMB=0.9970703125, forceMergeDeletesPctAllowed=2.487897048783277, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.35354646909200416
   [junit4]   2> 1500808 INFO  (coreLoadExecutor-2947-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1500808 INFO  (coreLoadExecutor-2947-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1500809 INFO  (coreLoadExecutor-2947-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=74.1171875, floorSegmentMB=0.7744140625, forceMergeDeletesPctAllowed=4.108848394977814, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.35848579947479964
   [junit4]   2> 1500809 INFO  (coreLoadExecutor-2947-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7e3954ec[collection1] main]
   [junit4]   2> 1500809 INFO  (coreLoadExecutor-2947-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-002/collection1/conf
   [junit4]   2> 1500810 INFO  (coreLoadExecutor-2947-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 1000ms
   [junit4]   2> 1500810 INFO  (coreLoadExecutor-2947-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1500822 INFO  (searcherExecutor-2948-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7e3954ec[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1500822 INFO  (qtp353127954-13418) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 1500822 INFO  (indexFetcher-2952-thread-1) [    x:collection1] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 1500822 INFO  (indexFetcher-2952-thread-1) [    x:collection1] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 1500822 INFO  (indexFetcher-2952-thread-1) [    x:collection1] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 1500822 INFO  (indexFetcher-2952-thread-1) [    x:collection1] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 1500822 INFO  (indexFetcher-2952-thread-1) [    x:collection1] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
   [junit4]   2> 1500822 INFO  (indexFetcher-2952-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1500822 INFO  (indexFetcher-2952-thread-1) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@3d13fd3a commitCommandVersion:0
   [junit4]   2> 1500827 INFO  (indexFetcher-2952-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6f409031[collection1] main]
   [junit4]   2> 1500827 INFO  (indexFetcher-2952-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1500828 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@4d2a2dbd{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1500828 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=466123618
   [junit4]   2> 1500828 INFO  (searcherExecutor-2948-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6f409031[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1500828 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 1500829 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2263b1e3: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@7f249039
   [junit4]   2> 1500834 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1500834 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@72ddb532: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@5dbd2814
   [junit4]   2> 1500837 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1500837 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@60882d81: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@17a91bcc
   [junit4]   2> 1500841 INFO  (coreCloseExecutor-2953-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@28807124
   [junit4]   2> 1500841 INFO  (coreCloseExecutor-2953-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=679506212
   [junit4]   2> 1500841 INFO  (coreCloseExecutor-2953-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1b05db3d: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@3a9213b4
   [junit4]   2> 1500857 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6cd8f708{/solr,null,UNAVAILABLE}
   [junit4]   2> 1500860 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@a6274a5{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1500860 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1822009639
   [junit4]   2> 1500860 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 1500860 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@cc7c730: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@4818923d
   [junit4]   2> 1500860 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1500861 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@661bd96c: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@5dbd2814
   [junit4]   2> 1500861 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1500861 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4e89ecd4: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@17a91bcc
   [junit4]   2> 1500864 INFO  (coreCloseExecutor-2955-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@73905b03
   [junit4]   2> 1500864 INFO  (coreCloseExecutor-2955-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1938840323
   [junit4]   2> 1500864 INFO  (coreCloseExecutor-2955-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1ad56924: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@7b685246
   [junit4]   2> 1500872 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@a332755{/solr,null,UNAVAILABLE}
   [junit4]   2> 1500875 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1500876 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@79931642{/solr,null,AVAILABLE}
   [junit4]   2> 1500876 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3aa7f59b{HTTP/1.1,[http/1.1]}{127.0.0.1:39666}
   [junit4]   2> 1500876 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.Server Started @1503089ms
   [junit4]   2> 1500876 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/collection1/data, hostContext=/solr, hostPort=39666}
   [junit4]   2> 1500877 ERROR (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1500879 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1500879 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 1500879 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1500879 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-04T21:48:09.801Z
   [junit4]   2> 1500880 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/solr.xml
   [junit4]   2> 1500883 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1500883 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1500884 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1500918 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd
   [junit4]   2> 1500926 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd
   [junit4]   2> 1500926 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd
   [junit4]   2> 1500927 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/.
   [junit4]   2> 1500927 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1500928 INFO  (coreLoadExecutor-2962-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1500941 INFO  (coreLoadExecutor-2962-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1500947 INFO  (coreLoadExecutor-2962-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1500948 INFO  (coreLoadExecutor-2962-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 1500948 INFO  (coreLoadExecutor-2962-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/./collection1, trusted=true
   [junit4]   2> 1500948 INFO  (coreLoadExecutor-2962-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd
   [junit4]   2> 1500948 INFO  (coreLoadExecutor-2962-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1500948 INFO  (coreLoadExecutor-2962-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/./collection1/data/]
   [junit4]   2> 1500949 INFO  (coreLoadExecutor-2962-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=2, maxMergeAtOnceExplicit=6, maxMergedSegmentMB=77.86328125, floorSegmentMB=0.9970703125, forceMergeDeletesPctAllowed=2.487897048783277, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.35354646909200416
   [junit4]   2> 1501005 INFO  (coreLoadExecutor-2962-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1501005 INFO  (coreLoadExecutor-2962-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1501006 INFO  (coreLoadExecutor-2962-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=74.1171875, floorSegmentMB=0.7744140625, forceMergeDeletesPctAllowed=4.108848394977814, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.35848579947479964
   [junit4]   2> 1501006 INFO  (coreLoadExecutor-2962-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@420607b0[collection1] main]
   [junit4]   2> 1501006 INFO  (coreLoadExecutor-2962-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/collection1/conf
   [junit4]   2> 1501006 INFO  (coreLoadExecutor-2962-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following config files: schema.xml
   [junit4]   2> 1501006 INFO  (coreLoadExecutor-2962-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1501006 INFO  (searcherExecutor-2963-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@420607b0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1501017 INFO  (qtp863784252-13459) [    x:collection1] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 1501018 INFO  (qtp863784252-13459) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{deleteByQuery=*:*} 0 0
   [junit4]   2> 1501018 INFO  (qtp863784252-13460) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1501018 INFO  (qtp863784252-13460) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@214a24f6 commitCommandVersion:0
   [junit4]   2> 1501023 INFO  (qtp863784252-13460) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6eb0d494[collection1] main]
   [junit4]   2> 1501024 INFO  (qtp863784252-13460) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1501025 INFO  (searcherExecutor-2963-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6eb0d494[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1501025 INFO  (qtp863784252-13460) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 6
   [junit4]   2> 1501026 INFO  (qtp863784252-13461) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=*:*&sort=id+desc&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 1501026 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.h.TestReplicationHandler Waited for 0ms and found 0 docs
   [junit4]   2> 1501030 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1501031 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@78a375e2{/solr,null,AVAILABLE}
   [junit4]   2> 1501031 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@9d32c0d{HTTP/1.1,[http/1.1]}{127.0.0.1:49567}
   [junit4]   2> 1501031 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.Server Started @1503244ms
   [junit4]   2> 1501032 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-002/collection1/data, hostContext=/solr, hostPort=49567}
   [junit4]   2> 1501032 ERROR (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1501032 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1501032 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 1501032 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1501032 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-04T21:48:09.954Z
   [junit4]   2> 1501035 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-002/solr.xml
   [junit4]   2> 1501038 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 1501038 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 1501039 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1501081 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd
   [junit4]   2> 1501090 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd
   [junit4]   2> 1501090 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd
   [junit4]   2> 1501091 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-002/.
   [junit4]   2> 1501091 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1501093 INFO  (coreLoadExecutor-2972-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1501108 INFO  (coreLoadExecutor-2972-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1501114 INFO  (coreLoadExecutor-2972-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1501115 INFO  (coreLoadExecutor-2972-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 1501115 INFO  (coreLoadExecutor-2972-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-002/./collection1, trusted=true
   [junit4]   2> 1501116 INFO  (coreLoadExecutor-2972-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd
   [junit4]   2> 1501116 INFO  (coreLoadExecutor-2972-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1501116 INFO  (coreLoadExecutor-2972-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-002/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-002/./collection1/data/]
   [junit4]   2> 1501117 INFO  (coreLoadExecutor-2972-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=2, maxMergeAtOnceExplicit=6, maxMergedSegmentMB=77.86328125, floorSegmentMB=0.9970703125, forceMergeDeletesPctAllowed=2.487897048783277, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.35354646909200416
   [junit4]   2> 1501156 INFO  (coreLoadExecutor-2972-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1501156 INFO  (coreLoadExecutor-2972-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1501157 INFO  (coreLoadExecutor-2972-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=74.1171875, floorSegmentMB=0.7744140625, forceMergeDeletesPctAllowed=4.108848394977814, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.35848579947479964
   [junit4]   2> 1501157 INFO  (coreLoadExecutor-2972-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@111292d9[collection1] main]
   [junit4]   2> 1501157 INFO  (coreLoadExecutor-2972-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-002/collection1/conf
   [junit4]   2> 1501158 INFO  (coreLoadExecutor-2972-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 1000ms
   [junit4]   2> 1501158 INFO  (coreLoadExecutor-2972-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1501159 INFO  (searcherExecutor-2973-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@111292d9[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1501173 INFO  (qtp1265301787-13477) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=*:*&sort=id+desc&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 1501173 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.h.TestReplicationHandler Waited for 0ms and found 0 docs
   [junit4]   2> 1501174 INFO  (qtp863784252-13462) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=getIndexVersion&qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 1501174 INFO  (qtp863784252-13463) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=getCommits&qt=/replication&wt=javabin&version=2&command=commits} status=0 QTime=0
   [junit4]   2> 1501187 INFO  (qtp863784252-13457) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1501192 INFO  (qtp863784252-13457) [    ] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1501194 INFO  (qtp863784252-13457) [    ] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 1501194 INFO  (qtp863784252-13457) [    ] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/./collection1
   [junit4]   2> 1501212 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@38ab33cd
   [junit4]   2> 1501212 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/./collection1/data/]
   [junit4]   2> 1501254 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1501254 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1501255 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2685b25e[collection1] main]
   [junit4]   2> 1501255 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_AE584CE27515D63E-001/solr-instance-001/collection1/conf
   [junit4]   2> 1501256 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following config files: schema.xml
   [junit4]   2> 1501256 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1501259 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=49, maxMergedSegmentMB=69.4169921875, floorSegmentMB=1.2412109375, forceMergeDeletesPctAllowed=8.19040782526387, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3784452969658093
   [junit4]   2> 1501259 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 1501260 INFO  (searcherExecutor-2978-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2685b25e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1501260 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@100af24[collection1] main]
   [junit4]   2> 1501260 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@7332e061
   [junit4]   2> 1501260 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1932714081
   [junit4]   2> 1501260 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@f1ba02f: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@689ace5a
   [junit4]   2> 1501261 INFO  (searcherExecutor-2978-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@100af24[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1501265 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={core=collection1&qt=/admin/cores&action=reload&wt=javabin&version=2} status=0 QTime=89
   [junit4]   2> 1501266 INFO  (qtp863784252-13460) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=getIndexVersion&qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 1501266 INFO  (qtp863784252-13462) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={_trace=getCommits&qt=/replication&wt=javabin&version=2&command=commits} status=0 QTime=0
   [junit4]   2> 1501267 INFO  (qtp863784252-13463) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[10]} 0 0
   [junit4]   2> 1501270 INFO  (qtp863784252-13464) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[20]} 0 2
   [junit4]   2> 1501271 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1501271 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@7c7e962c commitCommandVersion:0
   [junit4]   2> 1501272 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@cd5125f[collection1] main]
   [junit4]   2> 1501272 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1501273 INFO  (searcherExecutor-2978-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@cd5125f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C2)))}
   [junit4]   2> 1501273 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 1
   [junit4]   2> 1501273 INFO  (qtp863784252-13459) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=*:*&sort=id+desc&wt=javabin&version=2} hits=2 status=0 QTime=0
   [junit4]   2> 1501274 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.h.TestReplicationHandler Waited for 0ms and found 2 docs
   [junit4]   2> 1501274 INFO  (qtp1265301787-13478) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=*:*&sort=id+desc&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 1501274 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.h.TestReplicationHandler Waiting for 2 docs
   [junit4]   2> 1501376 INFO  (qtp1265301787-13479) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=*:*&sort=id+desc&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 1501376 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.h.TestReplicationHandler Waiting for 2 docs
   [junit4]   2> 1501477 INFO  (qtp1265301787-13480) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=*:*&sort=id+desc&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 1501477 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.h.TestReplicationHandler Waiting for 2 docs
   [junit4]   2> 1501578 INFO  (qtp1265301787-13481) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=*:*&sort=id+desc&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 1501578 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.h.TestReplicationHandler Waiting for 2 docs
   [junit4]   2> 1501624 INFO  (qtp863784252-13461) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 1501624 INFO  (indexFetcher-2977-thread-1) [    x:collection1] o.a.s.h.IndexFetcher Master's generation: 3
   [junit4]   2> 1501625 INFO  (indexFetcher-2977-thread-1) [    x:collection1] o.a.s.h.IndexFetcher Master's version: 1507153690193
   [junit4]   2> 1501625 INFO  (indexFetcher-2977-thread-1) [    x:collection1] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 1501625 INFO  (indexFetcher-2977-thread-1) [    x:collection1] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 1501625 INFO  (indexFetcher-2977-thread-1) [    x:collection1] o.a.s.h.IndexFetcher Starting replication process
   [junit4]   2> 1501627 INFO  (qtp863784252-13461) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&wt=javabin&version=2&command=filelist} status=0 QTime=1
   [junit4]   2> 1501627 INFO  (indexFetcher-2977-thread-1) [    x:collection1] o.a.s.h.IndexFetcher Number of files in latest index in master: 10
   [junit4]   2> 1501630 INFO  (indexFetcher-2977-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=229680935, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1501633 INFO  (indexFetcher-2977-thread-1) [    x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 1501633 INFO  (indexFetcher-2977-thread-1) [    x:collection1] o.a.s.h.IndexFetcher Starting download (fullCopy=false) to MockDirectoryWrapper(RAMDirectory@5ad19eb2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@79043286)
   [junit4]   2> 1501633 INFO  (qtp863784252-13462) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=_0.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 1501634 INFO  (qtp863784252-13463) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=_0.nvd&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 1501637 INFO  (qtp863784252-13464) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=_0.tip&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 1501638 INFO  (qtp863784252-13457) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=_0.fdx&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 1501639 INFO  (qtp863784252-13459) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=_0.doc&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 1501640 INFO  (qtp863784252-13460) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=_0.tim&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 1501640 INFO  (qtp863784252-13461) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=_0.fdt&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 1501641 INFO  (qtp863784252-13462) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=_0.fnm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 1501642 INFO  (qtp863784252-13463) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=_0.nvm&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 1501642 INFO  (qtp863784252-13464) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=3&qt=/replication&file=segments_3&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 1501643 INFO  (indexFetcher-2977-thread-1) [    x:collection1] o.a.s.h.IndexFetcher Total time taken for download (fullCopy=false,bytesDownloaded=1503) : 0 secs (null bytes/sec) to MockDirectoryWrapper(RAMDirectory@5ad19eb2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@79043286)
   [junit4]   2> 1501644 INFO  (indexFetcher-2977-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=54.041015625, floorSegmentMB=0.8642578125, forceMergeDeletesPctAllowed=11.80472124987806, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1663439213008874
   [junit4]   2> 1501647 INFO  (indexFetcher-2977-thread-1) [    x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 1501647 INFO  (indexFetcher-2977-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@9b5a984[collection1] main]
   [junit4]   2> 1501648 INFO  (searcherExecutor-2973-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@9b5a984[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C2)))}
   [junit4]   2> 1501679 INFO  (qtp1265301787-13482) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=*:*&sort=id+desc&wt=javabin&version=2} hits=2 status=0 QTime=0
   [junit4]   2> 1501679 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.h.TestReplicationHandler Waited for 400ms and found 2 docs
   [junit4]   2> 1501679 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.SolrTestCaseJ4 ###Ending doTestReplicateAfterCoreReload
   [junit4]   2> 1501680 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@3aa7f59b{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1501680 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=980837206
   [junit4]   2> 1501680 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 1501680 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4db30fc6: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@1e91ac1a
   [junit4]   2> 1501684 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1501684 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@220e3046: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@5dbd2814
   [junit4]   2> 1501686 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1501686 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[AE584CE27515D63E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@446e2522: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@17a91bcc
   [junit4]   2> 1501688 INFO  (coreCloseExecutor-2983-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@4056a276
   [junit4]   2> 1501688 INFO  (coreCloseExecutor-2983-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1079419510
   [junit4]   2> 1501688 INFO  (coreCloseExecutor-2983-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@601bf468: rootName = null, domain

[...truncated too long message...]

sUnloadDistributedZkTest, CollectionsAPIAsyncDistributedZkTest, AutoScalingHandlerTest, ZkCLITest, DistribDocExpirationUpdateProcessorTest, TestScoreJoinQPScore, TestSolrCoreSnapshots, SimplePostToolTest, TestXIncludeConfig, TestRecoveryHdfs, PrimitiveFieldTypeTest, TestConfigSetImmutable, RecoveryAfterSoftCommitTest, FastVectorHighlighterTest, TimeZoneUtilsTest, BlockJoinFacetRandomTest, OverseerTest, TestSlowCompositeReaderWrapper, TestQueryWrapperFilter, SolrInfoBeanTest, BufferStoreTest, SearchHandlerTest, RAMDirectoryFactoryTest, TestLFUCache, UniqFieldsUpdateProcessorFactoryTest, DistributedDebugComponentTest, TestFunctionQuery, TestRTimerTree, TestXmlQParserPlugin, SSLMigrationTest, TestManagedResourceStorage, AssignTest, TestCursorMarkWithoutUniqueKey, RecoveryZkTest, TestSha256AuthenticationProvider, ClusterStateUpdateTest, RestartWhileUpdatingTest, TestCloudRecovery, TestTestInjection, TestBinaryResponseWriter, CloudExitableDirectoryReaderTest, UUIDUpdateProcessorFallbackTest, CSVRequestHandlerTest, TestHalfAndHalfDocValues, TestPseudoReturnFields, BJQParserTest, CustomCollectionTest, InfixSuggestersTest, AnalysisAfterCoreReloadTest, ShardRoutingTest, ReplaceNodeTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestClassicSimilarityFactory, CoreAdminRequestStatusTest, DirectUpdateHandlerOptimizeTest, HdfsBasicDistributedZk2Test, TestLegacyFieldCache, TestUseDocValuesAsStored2, ExecutePlanActionTest, AtomicUpdateProcessorFactoryTest, TestSolrConfigHandlerConcurrent, TestTolerantSearch, SegmentsInfoRequestHandlerTest, SolrIndexSplitterTest, SyncSliceTest, ScriptEngineTest, SuggestComponentContextFilterQueryTest, AddBlockUpdateTest, MoveReplicaHDFSTest, DateFieldTest, TestRawTransformer, TestReplicationHandler, DistributedQueueTest, DistributedSuggestComponentTest, NodeLostTriggerTest, DocValuesTest, TestHashQParserPlugin, TestBinaryField, TestSweetSpotSimilarityFactory, TestWriterPerf, HdfsLockFactoryTest, TestLegacyNumericUtils, ParsingFieldUpdateProcessorsTest, SuggesterWFSTTest, UUIDFieldTest, TestSubQueryTransformerDistrib, TestCoreContainer, TestSolrCoreProperties, TestCloudPivotFacet, SolrCloudExampleTest, TestDFRSimilarityFactory, XmlUpdateRequestHandlerTest, SolrGraphiteReporterTest, SolrCoreTest, TestNoOpRegenerator, TestReversedWildcardFilterFactory, CurrencyFieldTypeTest, TestRestoreCore, GraphQueryTest, ClusterStateTest, DefaultValueUpdateProcessorTest, TestSchemaSimilarityResource, RequestHandlersTest, TestPolicyCloud, TestSolrDeletionPolicy2, TestSerializedLuceneMatchVersion, MultiThreadedOCPTest, HighlighterTest, TestDocBasedVersionConstraints, RulesTest, TestJmxIntegration, HighlighterConfigTest, TestRequestForwarding, SolrTestCaseJ4Test, TestOnReconnectListenerSupport, DistributedSpellCheckComponentTest, TestManagedStopFilterFactory, TestCodecSupport, TestRandomDVFaceting, TestStressRecovery, NotRequiredUniqueKeyTest, ResponseLogComponentTest, TestManagedSchemaAPI, TestSQLHandlerNonCloud, TestIntervalFaceting, TestStressCloudBlindAtomicUpdates, HealthCheckHandlerTest, DistributedFacetPivotLongTailTest, TestCustomSort, TestConfigSetsAPIExclusivity, TestDistributedStatsComponentCardinality, BasicAuthIntegrationTest, TestQuerySenderListener, TestMultiValuedNumericRangeQuery, MissingSegmentRecoveryTest, TestPhraseSuggestions, TestAuthenticationFramework, TestCloudInspectUtil, TestStressVersions, ChaosMonkeyNothingIsSafeTest, TestGraphMLResponseWriter, TestOrdValues, TestLockTree, QueryParsingTest, TestExportWriter, TestExclusionRuleCollectionAccess, DistributedFacetExistsSmallTest, TestBulkSchemaAPI, TestUpdate, TestSubQueryTransformer, OverseerTaskQueueTest, TestFreeTextSuggestions, TestSearcherReuse, DisMaxRequestHandlerTest, CollectionStateFormat2Test, MetricsConfigTest, SuggesterTest, TestReloadAndDeleteDocs, TestSolrCloudWithKerberosAlt, TestStressLiveNodes, TestIBSimilarityFactory, TestConfigsApi, OpenCloseCoreStressTest, FullSolrCloudDistribCmdsTest, TestRandomFaceting, LeaderElectionTest, TestZkChroot, ShardRoutingCustomTest, TestDistributedGrouping, TestFaceting, TestRecovery, TestHashPartitioner, TestRealTimeGet, TestJoin, HardAutoCommitTest, TestRangeQuery, ConvertedLegacyTest, TestSort, TestFiltering, TestLazyCores, ShowFileRequestHandlerTest, TestCoreDiscovery, TestExtendedDismaxParser, SpellCheckCollatorTest, SuggesterFSTTest, CoreAdminHandlerTest, TestFoldingMultitermQuery, TestStressLucene, TestTrie, SpatialFilterTest, PolyFieldTest, NoCacheHeaderTest, WordBreakSolrSpellCheckerTest, SchemaVersionSpecificBehaviorTest, SortByFunctionTest, TestRemoteStreaming, DocValuesMultiTest, DistanceFunctionTest, TestSolrDeletionPolicy1, XsltUpdateRequestHandlerTest, DebugComponentTest, CacheHeaderTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, LukeRequestHandlerTest, DirectSolrSpellCheckerTest, TestOmitPositions, TermsComponentTest, PathHierarchyTokenizerFactoryTest, TestIndexingPerformance, MoreLikeThisHandlerTest, RequiredFieldsTest, FieldAnalysisRequestHandlerTest, IndexSchemaRuntimeFieldTest, SolrPluginUtilsTest, IndexSchemaTest, TestCollationField, ReturnFieldsTest, UpdateRequestProcessorFactoryTest, TestCSVResponseWriter, BinaryUpdateRequestHandlerTest, TestPartialUpdateDeduplication, TestPHPSerializedResponseWriter, TestComponentsName, SearchHandlerTest, TestDistributedMissingSort, ActionThrottleTest, AliasIntegrationTest, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrBootstrapTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CdcrBootstrapTest -Dtests.seed=AE584CE27515D63E -Dtests.slow=true -Dtests.locale=es-CU -Dtests.timezone=Africa/Sao_Tome -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | CdcrBootstrapTest (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.CdcrBootstrapTest: 
   [junit4]    >    1) Thread[id=27281, name=Thread-6704, state=WAITING, group=TGRP-CdcrBootstrapTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.lang.Thread.join(Thread.java:1252)
   [junit4]    >         at java.lang.Thread.join(Thread.java:1326)
   [junit4]    >         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)
   [junit4]    >         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
   [junit4]    >         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   [junit4]    >    2) Thread[id=27285, name=ProcessThread(sid:0 cport:42976):, state=WAITING, group=TGRP-CdcrBootstrapTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)
   [junit4]    >    3) Thread[id=27282, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-CdcrBootstrapTest]
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    4) Thread[id=27283, name=SessionTracker, state=TIMED_WAITING, group=TGRP-CdcrBootstrapTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AE584CE27515D63E]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=27282, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-CdcrBootstrapTest]
   [junit4]    >         at sun.nio.ch.IOUtil.drain(Native Method)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:109)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AE584CE27515D63E]:0)
   [junit4] Completed [512/745 (4!)] on J0 in 122.33s, 3 tests, 3 errors <<< FAILURES!

[...truncated 47641 lines...]
BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:826: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:706: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:693: Source checkout is dirty (unversioned/missing files) after running tests!!! Offending files:
* lucene/licenses/morfologik-ukrainian-search-3.7.5.jar.sha1

Total time: 77 minutes 50 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1453 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1453/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.cloud.TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry

Error Message:
Captured an uncaught exception in thread: Thread[id=25503, name=OverseerAutoScalingTriggerThread-98770407157530626-dummy.host.com:8984_solr-n_0000000000, state=RUNNABLE, group=Overseer autoscaling triggers]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=25503, name=OverseerAutoScalingTriggerThread-98770407157530626-dummy.host.com:8984_solr-n_0000000000, state=RUNNABLE, group=Overseer autoscaling triggers]
	at __randomizedtesting.SeedInfo.seed([31BEEBE2FFCBB893:E761CAB8C9D6AB0B]:0)
Caused by: org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.auto_add_replicas
	at __randomizedtesting.SeedInfo.seed([31BEEBE2FFCBB893]:0)
	at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:107)
	at org.apache.solr.cloud.autoscaling.TriggerEventQueue.<init>(TriggerEventQueue.java:44)
	at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.<init>(ScheduledTriggers.java:398)
	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.add(ScheduledTriggers.java:149)
	at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:220)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.auto_add_replicas
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
	at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:93)
	at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:78)
	at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:105)
	... 5 more




Build Log:
[...truncated 13277 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestLeaderElectionZkExpiry
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLeaderElectionZkExpiry_31BEEBE2FFCBB893-001/init-core-data-001
   [junit4]   2> 2803362 WARN  (SUITE-TestLeaderElectionZkExpiry-seed#[31BEEBE2FFCBB893]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=5 numCloses=5
   [junit4]   2> 2803363 INFO  (SUITE-TestLeaderElectionZkExpiry-seed#[31BEEBE2FFCBB893]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 2803364 INFO  (SUITE-TestLeaderElectionZkExpiry-seed#[31BEEBE2FFCBB893]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2803364 INFO  (SUITE-TestLeaderElectionZkExpiry-seed#[31BEEBE2FFCBB893]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2803370 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testLeaderElectionWithZkExpiry
   [junit4]   2> 2803373 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1c2569ac, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 2803419 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c2569ac
   [junit4]   2> 2803426 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c2569ac
   [junit4]   2> 2803426 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c2569ac
   [junit4]   2> 2803427 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLeaderElectionZkExpiry_31BEEBE2FFCBB893-001/testLeaderElectionWithZkExpiry-solr-001
   [junit4]   2> 2803428 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2803428 INFO  (Thread-5763) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2803428 INFO  (Thread-5763) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2803429 ERROR (Thread-5763) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2803528 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [    ] o.a.s.c.ZkTestServer start zk server on port:64902
   [junit4]   2> 2803659 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [n:dummy.host.com:8984_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2803660 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [n:dummy.host.com:8984_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader dummy.host.com:8984_solr
   [junit4]   2> 2803661 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [n:dummy.host.com:8984_solr    ] o.a.s.c.Overseer Overseer (id=98770407157530626-dummy.host.com:8984_solr-n_0000000000) starting
   [junit4]   2> 2803669 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [n:dummy.host.com:8984_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/dummy.host.com:8984_solr
   [junit4]   2> 2803670 INFO  (zkCallback-4442-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2803674 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxnFactory Ignoring unexpected runtime exception
   [junit4]   2> java.nio.channels.CancelledKeyException
   [junit4]   2> 	at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73)
   [junit4]   2> 	at sun.nio.ch.SelectionKeyImpl.readyOps(SelectionKeyImpl.java:87)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:182)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2803675 WARN  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]-SendThread(127.0.0.1:64902)) [n:dummy.host.com:8984_solr    ] o.a.z.ClientCnxn Session 0x15ee72978f20002 for server 127.0.0.1/127.0.0.1:64902, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.io.IOException: Connection reset by peer
   [junit4]   2> 	at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
   [junit4]   2> 	at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
   [junit4]   2> 	at sun.nio.ch.IOUtil.read(IOUtil.java:192)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:68)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
   [junit4]   2> 2803776 WARN  (zkCallback-4442-thread-2-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@33e50aa0 name: ZooKeeperConnection Watcher:127.0.0.1:64902/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 2803776 WARN  (zkCallback-4442-thread-2-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 2805232 WARN  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]-SendThread(127.0.0.1:64902)) [n:dummy.host.com:8984_solr    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15ee72978f20002 has expired
   [junit4]   2> 2805232 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@33e50aa0 name: ZooKeeperConnection Watcher:127.0.0.1:64902/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
   [junit4]   2> 2805232 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
   [junit4]   2> 2805232 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr    ] o.a.s.c.Overseer Overseer (id=98770407157530626-dummy.host.com:8984_solr-n_0000000000) closing
   [junit4]   2> 2805232 WARN  (OverseerCollectionConfigSetProcessor-98770407157530626-dummy.host.com:8984_solr-n_0000000000) [n:dummy.host.com:8984_solr    ] o.a.s.c.OverseerTaskProcessor Overseer cannot talk to ZK
   [junit4]   2> 2805235 INFO  (OverseerStateUpdate-98770407157530626-dummy.host.com:8984_solr-n_0000000000) [n:dummy.host.com:8984_solr    ] o.a.s.c.Overseer Overseer Loop exiting : dummy.host.com:8984_solr
   [junit4]   2> 2805237 ERROR (OverseerExitThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.Overseer could not read the data
   [junit4]   2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:299)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2805276 WARN  (OverseerAutoScalingTriggerThread-98770407157530626-dummy.host.com:8984_solr-n_0000000000) [n:dummy.host.com:8984_solr    ] o.a.s.c.a.TriggerBase Exception checking ZK path /autoscaling/triggerState
   [junit4]   2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/triggerState
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:306)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:303)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:303)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:512)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:415)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.TriggerBase.<init>(TriggerBase.java:45)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.NodeLostTrigger.<init>(NodeLostTrigger.java:72)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.AutoScaling$TriggerFactory.create(AutoScaling.java:138)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.loadTriggers(OverseerTriggerThread.java:365)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.refreshAutoScalingConf(OverseerTriggerThread.java:312)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:137)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2805277 WARN  (zkCallback-4442-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> ???. 04, 2017 4:33:30 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[OverseerAutoScalingTriggerThread-98770407157530626-dummy.host.com:8984_solr-n_0000000000,5,Overseer autoscaling triggers]
   [junit4]   2> org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.auto_add_replicas
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([31BEEBE2FFCBB893]:0)
   [junit4]   2> 	at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:107)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.TriggerEventQueue.<init>(TriggerEventQueue.java:44)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.<init>(ScheduledTriggers.java:398)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.add(ScheduledTriggers.java:149)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:220)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.auto_add_replicas
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:93)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:78)
   [junit4]   2> 	at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:105)
   [junit4]   2> 	... 5 more
   [junit4]   2> 
   [junit4]   2> 2805288 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
   [junit4]   2> 2805290 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
   [junit4]   2> 2805290 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr    ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
   [junit4]   2> 2805291 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 2805294 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x15ee72978f20003 due to java.nio.channels.CancelledKeyException
   [junit4]   2> 2805394 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@33e50aa0 name: ZooKeeperConnection Watcher:127.0.0.1:64902/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 2805394 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 2806982 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-SendThread(127.0.0.1:64902)) [n:dummy.host.com:8984_solr    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15ee72978f20003 has expired
   [junit4]   2> 2806982 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@33e50aa0 name: ZooKeeperConnection Watcher:127.0.0.1:64902/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
   [junit4]   2> 2806982 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
   [junit4]   2> 2806982 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
   [junit4]   2> 2806983 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
   [junit4]   2> 2806983 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
   [junit4]   2> 2807082 ERROR (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr    ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /aliases.json
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.updateAliases(ZkStateReader.java:898)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:475)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController$1.command(ZkController.java:322)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168)
   [junit4]   2> 	at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 2807083 ERROR (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException: 
   [junit4]   2> 	at org.apache.solr.cloud.ZkController$1.command(ZkController.java:407)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168)
   [junit4]   2> 	at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /aliases.json
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.updateAliases(ZkStateReader.java:898)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:475)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController$1.command(ZkController.java:322)
   [junit4]   2> 	... 10 more
   [junit4]   2> 
   [junit4]   2> 2807083 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed
   [junit4]   2> 2807085 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Connected:true
   [junit4]   2> 2807087 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@33e50aa0 name: ZooKeeperConnection Watcher:127.0.0.1:64902/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 2807088 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 2808712 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-SendThread(127.0.0.1:64902)) [n:dummy.host.com:8984_solr    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15ee72978f20004 has expired
   [junit4]   2> 2808712 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@33e50aa0 name: ZooKeeperConnection Watcher:127.0.0.1:64902/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
   [junit4]   2> 2808712 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
   [junit4]   2> 2808712 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
   [junit4]   2> 2808714 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
   [junit4]   2> 2808714 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
   [junit4]   2> 2808813 ERROR (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:426)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController$1.command(ZkController.java:322)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168)
   [junit4]   2> 	at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> 
   [junit4]   2> 2808813 ERROR (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException: 
   [junit4]   2> 	at org.apache.solr.cloud.ZkController$1.command(ZkController.java:407)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168)
   [junit4]   2> 	at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:426)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController$1.command(ZkController.java:322)
   [junit4]   2> 	... 5 more
   [junit4]   2> 
   [junit4]   2> 2808813 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed
   [junit4]   2> 2808815 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Connected:true
   [junit4]   2> 2808822 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@33e50aa0 name: ZooKeeperConnection Watcher:127.0.0.1:64902/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 2808822 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 2809871 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-SendThread(127.0.0.1:64902)) [n:dummy.host.com:8984_solr    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15ee72978f20005 has expired
   [junit4]   2> 2809871 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@33e50aa0 name: ZooKeeperConnection Watcher:127.0.0.1:64902/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
   [junit4]   2> 2809871 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
   [junit4]   2> 2809871 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
   [junit4]   2> 2809873 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
   [junit4]   2> 2809873 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
   [junit4]   2> 2809981 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@33e50aa0 name: ZooKeeperConnection Watcher:127.0.0.1:64902/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 2809981 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 2811492 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-SendThread(127.0.0.1:64902)) [n:dummy.host.com:8984_solr    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15ee72978f20006 has expired
   [junit4]   2> 2811492 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@33e50aa0 name: ZooKeeperConnection Watcher:127.0.0.1:64902/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
   [junit4]   2> 2811492 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
   [junit4]   2> 2811492 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
   [junit4]   2> 2811493 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
   [junit4]   2> 2811493 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
   [junit4]   2> 2811592 ERROR (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:426)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController$1.command(ZkController.java:322)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168)
   [junit4]   2> 	at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> 
   [junit4]   2> 2811592 ERROR (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException: 
   [junit4]   2> 	at org.apache.solr.cloud.ZkController$1.command(ZkController.java:407)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168)
   [junit4]   2> 	at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:426)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController$1.command(ZkController.java:322)
   [junit4]   2> 	... 5 more
   [junit4]   2> 
   [junit4]   2> 2811592 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed
   [junit4]   2> 2811595 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Connected:true
   [junit4]   2> 2811603 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@33e50aa0 name: ZooKeeperConnection Watcher:127.0.0.1:64902/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 2811604 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 2813394 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-SendThread(127.0.0.1:64902)) [n:dummy.host.com:8984_solr    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15ee72978f20007 has expired
   [junit4]   2> 2813394 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@33e50aa0 name: ZooKeeperConnection Watcher:127.0.0.1:64902/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
   [junit4]   2> 2813395 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
   [junit4]   2> 2813395 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
   [junit4]   2> 2813397 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
   [junit4]   2> 2813397 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
   [junit4]   2> 2813495 ERROR (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:426)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController$1.command(ZkController.java:322)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168)
   [junit4]   2> 	at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> 
   [junit4]   2> 2813495 ERROR (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException: 
   [junit4]   2> 	at org.apache.solr.cloud.ZkController$1.command(ZkController.java:407)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168)
   [junit4]   2> 	at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:426)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController$1.command(ZkController.java:322)
   [junit4]   2> 	... 5 more
   [junit4]   2> 
   [junit4]   2> 2813496 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed
   [junit4]   2> 2813499 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Connected:true
   [junit4]   2> 2813499 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@33e50aa0 name: ZooKeeperConnection Watcher:127.0.0.1:64902/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 2813500 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 2815184 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-SendThread(127.0.0.1:64902)) [n:dummy.host.com:8984_solr    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15ee72978f20008 has expired
   [junit4]   2> 2815184 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@33e50aa0 name: ZooKeeperConnection Watcher:127.0.0.1:64902/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
   [junit4]   2> 2815184 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
   [junit4]   2> 2815184 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
   [junit4]   2> 2815185 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
   [junit4]   2> 2815185 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
   [junit4]   2> 2815284 ERROR (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:426)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController$1.command(ZkController.java:322)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168)
   [junit4]   2> 	at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> 
   [junit4]   2> 2815284 ERROR (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException: 
   [junit4]   2> 	at org.apache.solr.cloud.ZkController$1.command(ZkController.java:407)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168)
   [junit4]   2> 	at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:426)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController$1.command(ZkController.java:322)
   [junit4]   2> 	... 5 more
   [junit4]   2> 
   [junit4]   2> 2815285 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed
   [junit4]   2> 2815287 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Connected:true
   [junit4]   2> 2815291 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader dummy.host.com:8984_solr
   [junit4]   2> 2815293 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.Overseer Overseer (id=98770407157530634-dummy.host.com:8984_solr-n_0000000001) starting
   [junit4]   2> 2815294 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [n:dummy.host.com:8984_solr    ] o.a.s.c.TestLeaderElectionZkExpiry Time=1294496485260407 Overseer leader is = dummy.host.com:8984_solr
   [junit4]   2> 2815299 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/dummy.host.com:8984_solr
   [junit4]   2> 2815299 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.Overseer Overseer (id=98770407157530634-dummy.host.com:8984_solr-n_0000000001) closing
   [junit4]   2> 2815300 INFO  (OverseerStateUpdate-98770407157530634-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr    ] o.a.s.c.Overseer According to ZK I (id=98770407157530634-dummy.host.com:8984_solr-n_0000000001) am no longer a leader.
   [junit4]   2> 2815302 INFO  (OverseerStateUpdate-98770407157530634-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr    ] o.a.s.c.Overseer Overseer Loop exiting : dummy.host.com:8984_solr
   [junit4]   2> 2815302 INFO  (OverseerCollectionConfigSetProcessor-98770407157530634-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr    ] o.a.s.c.OverseerTaskProcessor According to ZK I (id=98770407157530634-dummy.host.com:8984_solr-n_0000000001) am no longer a leader.
   [junit4]   2> 2815304 ERROR (OverseerCollectionConfigSetProcessor-98770407157530634-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr    ] o.a.s.c.OverseerTaskProcessor Unable to prioritize overseer 
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.lang.Object.wait(Native Method)
   [junit4]   2> 	at java.lang.Object.wait(Object.java:502)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1097)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerNodePrioritizer.prioritizeOverseerNodes(OverseerNodePrioritizer.java:61)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:178)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2815307 WARN  (OverseerExitThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.Overseer I'm exiting, but I'm still the leader
   [junit4]   2> 2815307 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Reconnected to ZooKeeper
   [junit4]   2> 2815307 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Connected:true
   [junit4]   2> 2815308 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15ee72978f2000a, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2816808 WARN  (zkCallback-4442-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2816808 WARN  (zkCallback-4442-thread-6-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2816809 ERROR (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1532)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:340)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:337)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:337)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController$1.command(ZkController.java:347)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168)
   [junit4]   2> 	at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> 
   [junit4]   2> 2816810 ERROR (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException: 
   [junit4]   2> 	at org.apache.solr.cloud.ZkController$1.command(ZkController.java:407)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168)
   [junit4]   2> 	at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1532)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:340)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:337)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:337)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController$1.command(ZkController.java:347)
   [junit4]   2> 	... 5 more
   [junit4]   2> 
   [junit4]   2> 2816810 WARN  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed
   [junit4]   2> 2816808 ERROR (OverseerExitThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.Overseer Could not delete my leader node 
   [junit4]   2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:246)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:243)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:243)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:310)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2816814 INFO  (zkCallback-4442-thread-5-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.c.ConnectionManager Connected:true
   [junit4]   2> 2816814 WARN  (OverseerExitThread) [n:dummy.host.com:8984_solr    ] o.a.s.c.Overseer Unable to rejoinElection 
   [junit4]   2> org.apache.solr.common.SolrException: Unable to rejoin election
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:1924)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:324)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect
   [junit4]   2> 	at org.apache.solr.cloud.OverseerElectionContext.<init>(ElectionContext.java:772)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerElectionContext.copy(ElectionContext.java:815)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:385)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:1921)
   [junit4]   2> 	... 2 more
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:93)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:78)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerElectionContext.<init>(ElectionContext.java:770)
   [junit4]   2> 	... 5 more
   [junit4]   2> 2816814 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [n:dummy.host.com:8984_solr    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=959856624
   [junit4]   2> 2816815 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [n:dummy.host.com:8984_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2816815 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [n:dummy.host.com:8984_solr    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@892c757: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@5b81e7bc
   [junit4]   2> 2816818 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [n:dummy.host.com:8984_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2816818 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [n:dummy.host.com:8984_solr    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@42733cb6: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@49ffb006
   [junit4]   2> 2816821 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [n:dummy.host.com:8984_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2816821 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [n:dummy.host.com:8984_solr    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@50c718ed: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@5aa407f0
   [junit4]   2> 2816821 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [n:dummy.host.com:8984_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2816821 ERROR (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [n:dummy.host.com:8984_solr    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2816822 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [n:dummy.host.com:8984_solr    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:64902 64902
   [junit4]   2> 2821884 INFO  (Thread-5763) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:64902 64902
   [junit4]   2> 2821891 WARN  (Thread-5763) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	4	/solr/aliases.json
   [junit4]   2> 	4	/solr/clusterprops.json
   [junit4]   2> 	3	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	4	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	4	/solr/collections
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 
   [junit4]   2> 2821892 INFO  (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[31BEEBE2FFCBB893]) [n:dummy.host.com:8984_solr    ] o.a.s.SolrTestCaseJ4 ###Ending testLeaderElectionWithZkExpiry
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLeaderElectionZkExpiry -Dtests.method=testLeaderElectionWithZkExpiry -Dtests.seed=31BEEBE2FFCBB893 -Dtests.slow=true -Dtests.locale=mk-MK -Dtests.timezone=Asia/Tashkent -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   18.5s J0 | TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=25503, name=OverseerAutoScalingTriggerThread-98770407157530626-dummy.host.com:8984_solr-n_0000000000, state=RUNNABLE, group=Overseer autoscaling triggers]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([31BEEBE2FFCBB893:E761CAB8C9D6AB0B]:0)
   [junit4]    > Caused by: org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.auto_add_replicas
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([31BEEBE2FFCBB893]:0)
   [junit4]    > 	at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:107)
   [junit4]    > 	at org.apache.solr.cloud.autoscaling.TriggerEventQueue.<init>(TriggerEventQueue.java:44)
   [junit4]    > 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.<init>(ScheduledTriggers.java:398)
   [junit4]    > 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.add(ScheduledTriggers.java:149)
   [junit4]    > 	at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:220)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.auto_add_replicas
   [junit4]    > 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]    > 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]    > 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1102)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
   [junit4]    > 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
   [junit4]    > 	at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:93)
   [junit4]    > 	at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:78)
   [junit4]    > 	at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:105)
   [junit4]    > 	... 5 more
   [junit4]   2> 2821900 INFO  (SUITE-TestLeaderElectionZkExpiry-seed#[31BEEBE2FFCBB893]-worker) [n:dummy.host.com:8984_solr    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 2821900 INFO  (SUITE-TestLeaderElectionZkExpiry-seed#[31BEEBE2FFCBB893]-worker) [n:dummy.host.com:8984_solr    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=959856624
   [junit4]   2> 2821900 INFO  (SUITE-TestLeaderElectionZkExpiry-seed#[31BEEBE2FFCBB893]-worker) [n:dummy.host.com:8984_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2821900 INFO  (SUITE-TestLeaderElectionZkExpiry-seed#[31BEEBE2FFCBB893]-worker) [n:dummy.host.com:8984_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2821901 INFO  (SUITE-TestLeaderElectionZkExpiry-seed#[31BEEBE2FFCBB893]-worker) [n:dummy.host.com:8984_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2821901 INFO  (SUITE-TestLeaderElectionZkExpiry-seed#[31BEEBE2FFCBB893]-worker) [n:dummy.host.com:8984_solr    ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=true): {}, locale=mk-MK, timezone=Asia/Tashkent
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=92765040,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [TestTolerantSearch, TestBinaryField, TestShortCircuitedRequests, TestCryptoKeys, DistributedDebugComponentTest, TestSolrCoreProperties, TestFreeTextSuggestions, TestSweetSpotSimilarityFactory, TestWriterPerf, HealthCheckHandlerTest, AnalyticsQueryTest, TestRestoreCore, MoveReplicaHDFSTest, TestCoreContainer, TestSolrQueryResponse, TestBackupRepositoryFactory, BooleanFieldTest, TestScoreJoinQPScore, SpatialFilterTest, TestUtils, TestLeaderInitiatedRecoveryThread, SortByFunctionTest, TestSlowCompositeReaderWrapper, BlockJoinFacetRandomTest, SSLMigrationTest, SolrCloudExampleTest, CdcrRequestHandlerTest, OverseerTest, TestRandomDVFaceting, PeerSyncTest, TestExportWriter, ResponseHeaderTest, TestLMDirichletSimilarityFactory, HdfsDirectoryFactoryTest, ScriptEngineTest, ConnectionReuseTest, TestSolrCloudWithKerberosAlt, TestDistributedSearch, CoreAdminRequestStatusTest, ConnectionManagerTest, TestSurroundQueryParser, AutoAddReplicasIntegrationTest, DirectSolrConnectionTest, RAMDirectoryFactoryTest, TestHttpShardHandlerFactory, V2ApiIntegrationTest, ClusterStateUpdateTest, TestAtomicUpdateErrorCases, UUIDFieldTest, TestUseDocValuesAsStored2, TestXIncludeConfig, TestFuzzyAnalyzedSuggestions, TestMissingGroups, DirectoryFactoryTest, DistributedFacetPivotSmallTest, TestExtendedDismaxParser, NumericFieldsTest, PKIAuthenticationIntegrationTest, TestSchemalessBufferedUpdates, DocValuesNotIndexedTest, TestBlendedInfixSuggestions, InfoHandlerTest, TestMacroExpander, VersionInfoTest, DistanceFunctionTest, TestSkipOverseerOperations, CurrencyFieldTypeTest, ResponseLogComponentTest, HdfsRecoverLeaseTest, ResourceLoaderTest, TestMaxTokenLenTokenizer, HLLSerializationTest, DeleteLastCustomShardedReplicaTest, TestConfigSetsAPIExclusivity, TestSha256AuthenticationProvider, TestNestedDocsSort, AtomicUpdateProcessorFactoryTest, TestXmlQParserPlugin, DeleteStatusTest, LeaderElectionTest, TestTolerantUpdateProcessorCloud, TestFieldCacheSort, TestEmbeddedSolrServerAdminHandler, TestRandomFlRTGCloud, CloudMLTQParserTest, TestSolrXml, TestCodecSupport, HdfsCollectionsAPIDistributedZkTest, DistributedSpellCheckComponentTest, TestBulkSchemaConcurrent, DistribDocExpirationUpdateProcessorTest, DefaultValueUpdateProcessorTest, TestNumericRangeQuery64, SliceStateTest, TestImpersonationWithHadoopAuth, TestSolrCoreParser, NodeLostTriggerTest, TestManagedResourceStorage, TlogReplayBufferedWhileIndexingTest, SegmentsInfoRequestHandlerTest, URLClassifyProcessorTest, TestConfigReload, GraphQueryTest, TestOrdValues, HttpTriggerListenerTest, TestSQLHandlerNonCloud, SuggesterWFSTTest, TestStandardQParsers, TestTestInjection, TestExclusionRuleCollectionAccess, SolrCoreTest, TestQuerySenderNoQuery, InfixSuggestersTest, TestReloadAndDeleteDocs, TestLFUCache, TestQuerySenderListener, TestUpdate, ChaosMonkeyNothingIsSafeTest, TestTrieFacet, AlternateDirectoryTest, HighlighterMaxOffsetTest, HdfsRestartWhileUpdatingTest, DirectUpdateHandlerOptimizeTest, LargeFieldTest, AssignTest, TestFieldTypeCollectionResource, ZkControllerTest, TestSubQueryTransformerDistrib, SolrMetricsIntegrationTest, TestAuthenticationFramework, TestExactStatsCache, BinaryUpdateRequestHandlerTest, TestConfigsApi, ExplicitHLLTest, SpellCheckCollatorWithCollapseTest, AddBlockUpdateTest, DistributedQueryComponentCustomSortTest, TestInPlaceUpdatesStandalone, TestHighlightDedupGrouping, TestRequestForwarding, SubstringBytesRefFilterTest, XmlUpdateRequestHandlerTest, XsltUpdateRequestHandlerTest, AutoScalingHandlerTest, HdfsThreadLeakTest, TestLegacyNumericUtils, RecoveryAfterSoftCommitTest, TestPushWriter, SolrMetricManagerTest, TestSubQueryTransformer, TestLegacyFieldCache, TestSerializedLuceneMatchVersion, TestLegacyNumericRangeQueryBuilder, TestReplicationHandlerBackup, AutoscalingHistoryHandlerTest, TestFilteredDocIdSet, TestPhraseSuggestions, DirectUpdateHandlerTest, TestSimpleTextCodec, DocValuesMissingTest, SparseHLLTest, LeaderInitiatedRecoveryOnShardRestartTest, TestFunctionQuery, SolrIndexSplitterTest, ZkStateReaderTest, PrimitiveFieldTypeTest, CollectionStateFormat2Test, TestNoOpRegenerator, CircularListTest, TestCustomStream, TestCursorMarkWithoutUniqueKey, TestOmitPositions, TestReversedWildcardFilterFactory, TestHdfsUpdateLog, SimpleCollectionCreateDeleteTest, TestManagedSynonymGraphFilterFactory, AliasIntegrationTest, SimplePostToolTest, BasicDistributedZkTest, BasicDistributedZk2Test, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, SyncSliceTest, OpenCloseCoreStressTest, LeaderElectionIntegrationTest, ShardRoutingTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, TestReplicationHandler, ZkSolrClientTest, TestZkChroot, ZkCLITest, ShardRoutingCustomTest, TestFaceting, TestHashPartitioner, TermVectorComponentDistributedTest, TestStressReorder, TestStressVersions, HardAutoCommitTest, SimpleFacetsTest, SpellCheckComponentTest, TestGroupingSearch, SolrCmdDistributorTest, QueryElevationComponentTest, TestSort, TestLazyCores, TestIndexSearcher, HighlighterTest, SoftAutoCommitTest, PathHierarchyTokenizerFactoryTest, TestIndexingPerformance, TestSolrQueryParser, RegexBoostProcessorTest, TestJmxIntegration, ReturnFieldsTest, QueryParsingTest, TestAnalyzedSuggestions, UniqFieldsUpdateProcessorFactoryTest, TestPartialUpdateDeduplication, TestPHPSerializedResponseWriter, CSVRequestHandlerTest, TestComponentsName, SearchHandlerTest, HighlighterConfigTest, UpdateParamsTest, CopyFieldTest, SolrIndexConfigTest, TestStressRecovery, BaseCdcrDistributedZkTest, CdcrBootstrapTest, ChaosMonkeySafeLeaderWithPullReplicasTest, CollectionsAPIAsyncDistributedZkTest, DeleteNodeTest, DeleteReplicaTest, DistribJoinFromCollectionTest, ForceLeaderTest, MissingSegmentRecoveryTest, MultiThreadedOCPTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, PeerSyncReplicationTest, SolrCLIZkUtilsTest, SolrXmlInZkTest, TestCloudInspectUtil, TestCloudPivotFacet, TestCloudPseudoReturnFields, TestCloudRecovery, TestClusterProperties, TestCollectionsAPIViaSolrCloudCluster, TestDistribDocBasedVersion, TestLeaderElectionWithEmptyReplica, TestLeaderElectionZkExpiry]
   [junit4] Completed [547/745 (1!)] on J0 in 18.57s, 1 test, 1 error <<< FAILURES!

[...truncated 43623 lines...]
-ecj-javadoc-lint-src:
    [mkdir] Created dir: /var/tmp/ecj112807485
 [ecj-lint] Compiling 1149 source files to /var/tmp/ecj112807485
 [ecj-lint] invalid Class-Path header in manifest of jar file: /export/home/jenkins/.ivy2/cache/org.restlet.jee/org.restlet/jars/org.restlet-2.3.0.jar
 [ecj-lint] invalid Class-Path header in manifest of jar file: /export/home/jenkins/.ivy2/cache/org.restlet.jee/org.restlet.ext.servlet/jars/org.restlet.ext.servlet-2.3.0.jar
 [ecj-lint] ----------
 [ecj-lint] 1. ERROR in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/cloud/AddReplicaCmd.java (at line 27)
 [ecj-lint] 	import java.util.concurrent.atomic.AtomicBoolean;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import java.util.concurrent.atomic.AtomicBoolean is never used
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 2. ERROR in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java (at line 26)
 [ecj-lint] 	import com.google.common.base.Preconditions;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.google.common.base.Preconditions is never used
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 3. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/core/CoreContainer.java (at line 1033)
 [ecj-lint] 	core = new SolrCore(this, dcore, coreConfig);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'core' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 4. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java (at line 234)
 [ecj-lint] 	dir = new BlockDirectory(path, hdfsDir, cache, null, blockCacheReadEnabled, false, cacheMerges, cacheReadOnce);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'dir' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 5. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (at line 121)
 [ecj-lint] 	reader = cfiltfac.create(reader);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'reader' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] 6. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (at line 145)
 [ecj-lint] 	return namedList;
 [ecj-lint] 	^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'listBasedTokenStream' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 7. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java (at line 1285)
 [ecj-lint] 	DirectoryReader reader = s==null ? null : s.get().getIndexReader();
 [ecj-lint] 	                ^^^^^^
 [ecj-lint] Resource leak: 'reader' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 8. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java (at line 517)
 [ecj-lint] 	ParallelStream parallelStream = new ParallelStream(zk, collection, tupleStream, numWorkers, comp);
 [ecj-lint] 	               ^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'parallelStream' is never closed
 [ecj-lint] ----------
 [ecj-lint] 9. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java (at line 743)
 [ecj-lint] 	ParallelStream parallelStream = new ParallelStream(zkHost, collection, tupleStream, numWorkers, comp);
 [ecj-lint] 	               ^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'parallelStream' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 10. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java (at line 578)
 [ecj-lint] 	tvWindowStream = new OffsetWindowTokenFilter(tvStream);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'tvWindowStream' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 11. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (at line 925)
 [ecj-lint] 	fastForRandomSet = new HashDocSet(sset.getDocs(), 0, sset.size());
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'fastForRandomSet' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 12. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/response/SmileResponseWriter.java (at line 33)
 [ecj-lint] 	new SmileWriter(out, request, response).writeResponse();
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 13. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java (at line 146)
 [ecj-lint] 	ratesJsonStream = resourceLoader.openResource(ratesFileLocation);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'ratesJsonStream' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 14. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java (at line 1315)
 [ecj-lint] 	UninvertingReader uninvertingReader = new UninvertingReader(new ReaderWrapper(searcher.getSlowAtomicReader(), collapseField), mapping);
 [ecj-lint] 	                  ^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'uninvertingReader' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 15. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 145)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] 16. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 154)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] 17. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 163)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 18. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java (at line 79)
 [ecj-lint] 	SolrIndexSearcher is = (SolrIndexSearcher) o;
 [ecj-lint] 	                  ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 19. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java (at line 79)
 [ecj-lint] 	SolrIndexSearcher is = (SolrIndexSearcher) o;
 [ecj-lint] 	                  ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 20. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java (at line 291)
 [ecj-lint] 	throw new SolrException(ErrorCode.SERVER_ERROR, "Error occurred while loading solr.xml from zookeeper", e);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'loader' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 21. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/update/CdcrTransactionLog.java (at line 98)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 22. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (at line 191)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 23. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (at line 261)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 24. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 228)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] 25. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 228)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] 26. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 260)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 27. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 275)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 28. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 529)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 29. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (at line 655)
 [ecj-lint] 	BAOS bos =  new BAOS();
 [ecj-lint] 	     ^^^
 [ecj-lint] Resource leak: 'bos' is never closed
 [ecj-lint] ----------
 [ecj-lint] 30. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (at line 655)
 [ecj-lint] 	BAOS bos =  new BAOS();
 [ecj-lint] 	     ^^^
 [ecj-lint] Resource leak: 'bos' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 31. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/java/org/apache/solr/util/SolrLogLayout.java (at line 149)
 [ecj-lint] 	SolrQueryRequest req = requestInfo == null ? null : requestInfo.getReq();
 [ecj-lint] 	                 ^^^
 [ecj-lint] Resource leak: 'req' is never closed
 [ecj-lint] ----------
 [ecj-lint] 31 problems (2 errors, 29 warnings)

BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:826: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:101: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build.xml:689: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/common-build.xml:2081: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/common-build.xml:2120: Compile failed; see the compiler error output for details.

Total time: 93 minutes 31 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1452 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1452/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

2 tests failed.
FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication

Error Message:
Index: 0, Size: 0

Stack Trace:
java.lang.IndexOutOfBoundsException: Index: 0, Size: 0
	at __randomizedtesting.SeedInfo.seed([7C1E330DBD08839F:685668589E0F3E81]:0)
	at java.util.ArrayList.rangeCheck(ArrayList.java:653)
	at java.util.ArrayList.get(ArrayList.java:429)
	at org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication(TestReplicationHandler.java:561)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testExecutorStream

Error Message:
Error from server at http://127.0.0.1:33029/solr/workQueue_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/workQueue_shard2_replica_n2/update. Reason: <pre>    Can not find: /solr/workQueue_shard2_replica_n2/update</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at http://127.0.0.1:33029/solr/workQueue_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/workQueue_shard2_replica_n2/update. Reason:
<pre>    Can not find: /solr/workQueue_shard2_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([EAD91D9727B41A1:2C6D102251116BB1]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:539)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:993)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
	at org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testExecutorStream(StreamExpressionTest.java:7471)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:33029/solr/workQueue_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/workQueue_shard2_replica_n2/update. Reason:
<pre>    Can not find: /solr/workQueue_shard2_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:591)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:516)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more




Build Log:
[...truncated 12564 lines...]
   [junit4] Suite: org.apache.solr.handler.TestReplicationHandler
   [junit4]   2> 1856555 INFO  (SUITE-TestReplicationHandler-seed#[7C1E330DBD08839F]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/init-core-data-001
   [junit4]   2> 1856555 WARN  (SUITE-TestReplicationHandler-seed#[7C1E330DBD08839F]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=16 numCloses=16
   [junit4]   2> 1856556 INFO  (SUITE-TestReplicationHandler-seed#[7C1E330DBD08839F]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 1856557 INFO  (SUITE-TestReplicationHandler-seed#[7C1E330DBD08839F]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 1856558 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.SolrTestCaseJ4 ###Starting doTestStressReplication
   [junit4]   2> 1856558 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-001/collection1
   [junit4]   2> 1856561 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1856561 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@746b077f{/solr,null,AVAILABLE}
   [junit4]   2> 1856562 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2f68bc42{HTTP/1.1,[http/1.1]}{127.0.0.1:43091}
   [junit4]   2> 1856562 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.Server Started @1860025ms
   [junit4]   2> 1856562 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-001/collection1/data, hostContext=/solr, hostPort=43091}
   [junit4]   2> 1856563 ERROR (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1856563 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1856563 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 1856563 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1856563 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-03T22:13:27.794Z
   [junit4]   2> 1856563 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-001/solr.xml
   [junit4]   2> 1856566 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1856596 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc
   [junit4]   2> 1856602 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc
   [junit4]   2> 1856602 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc
   [junit4]   2> 1856604 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-001/.
   [junit4]   2> 1856604 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1856604 INFO  (coreLoadExecutor-3880-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1856612 INFO  (coreLoadExecutor-3880-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1856625 INFO  (coreLoadExecutor-3880-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1856627 INFO  (coreLoadExecutor-3880-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 1856627 INFO  (coreLoadExecutor-3880-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-001/./collection1, trusted=true
   [junit4]   2> 1856627 INFO  (coreLoadExecutor-3880-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc
   [junit4]   2> 1856627 INFO  (coreLoadExecutor-3880-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1856627 INFO  (coreLoadExecutor-3880-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-001/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-001/./collection1/data/]
   [junit4]   2> 1856628 INFO  (coreLoadExecutor-3880-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=75.7265625, floorSegmentMB=0.65234375, forceMergeDeletesPctAllowed=11.494274922496768, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1856660 INFO  (coreLoadExecutor-3880-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1856660 INFO  (coreLoadExecutor-3880-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1856661 INFO  (coreLoadExecutor-3880-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=54.5, floorSegmentMB=1.8427734375, forceMergeDeletesPctAllowed=20.004781891310305, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1856661 INFO  (coreLoadExecutor-3880-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@688fe321[collection1] main]
   [junit4]   2> 1856661 INFO  (coreLoadExecutor-3880-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-001/collection1/conf
   [junit4]   2> 1856662 INFO  (coreLoadExecutor-3880-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following config files: schema.xml,xslt/dummy.xsl
   [junit4]   2> 1856662 INFO  (coreLoadExecutor-3880-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1856662 INFO  (searcherExecutor-3881-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@688fe321[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1856663 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/collection1
   [junit4]   2> 1856665 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1856666 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2480f826{/solr,null,AVAILABLE}
   [junit4]   2> 1856667 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2076d78a{HTTP/1.1,[http/1.1]}{127.0.0.1:39225}
   [junit4]   2> 1856667 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.Server Started @1860130ms
   [junit4]   2> 1856667 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/collection1/data, hostContext=/solr, hostPort=39225}
   [junit4]   2> 1856667 ERROR (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1856667 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1856667 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 1856667 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1856667 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-03T22:13:27.898Z
   [junit4]   2> 1856668 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/solr.xml
   [junit4]   2> 1856671 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1856698 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc
   [junit4]   2> 1856705 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc
   [junit4]   2> 1856705 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc
   [junit4]   2> 1856706 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/.
   [junit4]   2> 1856706 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1856707 INFO  (coreLoadExecutor-3890-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1856716 INFO  (coreLoadExecutor-3890-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1856720 INFO  (coreLoadExecutor-3890-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1856721 INFO  (coreLoadExecutor-3890-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 1856722 INFO  (coreLoadExecutor-3890-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/./collection1, trusted=true
   [junit4]   2> 1856722 INFO  (coreLoadExecutor-3890-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc
   [junit4]   2> 1856722 INFO  (coreLoadExecutor-3890-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1856722 INFO  (coreLoadExecutor-3890-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/./collection1/data/]
   [junit4]   2> 1856732 INFO  (coreLoadExecutor-3890-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=75.7265625, floorSegmentMB=0.65234375, forceMergeDeletesPctAllowed=11.494274922496768, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1856769 INFO  (coreLoadExecutor-3890-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1856769 INFO  (coreLoadExecutor-3890-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1856769 INFO  (coreLoadExecutor-3890-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=54.5, floorSegmentMB=1.8427734375, forceMergeDeletesPctAllowed=20.004781891310305, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1856770 INFO  (coreLoadExecutor-3890-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@488ca840[collection1] main]
   [junit4]   2> 1856770 INFO  (coreLoadExecutor-3890-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/collection1/conf
   [junit4]   2> 1856771 INFO  (coreLoadExecutor-3890-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 1000ms
   [junit4]   2> 1856771 INFO  (coreLoadExecutor-3890-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1856772 INFO  (searcherExecutor-3891-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@488ca840[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1856774 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@2076d78a{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1856774 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1495668197
   [junit4]   2> 1856774 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 1856774 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1d2cce2c: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@63e86dc9
   [junit4]   2> 1856779 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1856780 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6c49365a: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@1691e6c9
   [junit4]   2> 1856782 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1856782 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@60363d98: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@1736b3c5
   [junit4]   2> 1856783 INFO  (coreCloseExecutor-3896-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@651450b8
   [junit4]   2> 1856784 INFO  (coreCloseExecutor-3896-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1695830200
   [junit4]   2> 1856784 INFO  (coreCloseExecutor-3896-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3aa8fe75: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@661840ee
   [junit4]   2> 1856791 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2480f826{/solr,null,UNAVAILABLE}
   [junit4]   2> 1856793 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1856794 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@327c1817{/solr,null,AVAILABLE}
   [junit4]   2> 1856794 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4389c975{HTTP/1.1,[http/1.1]}{127.0.0.1:48314}
   [junit4]   2> 1856794 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.Server Started @1860258ms
   [junit4]   2> 1856794 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/collection1/data, hostContext=/solr, hostPort=48314}
   [junit4]   2> 1856795 ERROR (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1856795 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1856795 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 1856795 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1856795 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-03T22:13:28.026Z
   [junit4]   2> 1856796 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/solr.xml
   [junit4]   2> 1856799 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1856829 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc
   [junit4]   2> 1856833 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc
   [junit4]   2> 1856833 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc
   [junit4]   2> 1856833 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/.
   [junit4]   2> 1856833 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1856838 INFO  (coreLoadExecutor-3903-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1856847 INFO  (coreLoadExecutor-3903-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1856851 INFO  (coreLoadExecutor-3903-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1856852 INFO  (coreLoadExecutor-3903-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 1856852 INFO  (coreLoadExecutor-3903-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/./collection1, trusted=true
   [junit4]   2> 1856853 INFO  (coreLoadExecutor-3903-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc
   [junit4]   2> 1856853 INFO  (coreLoadExecutor-3903-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1856853 INFO  (coreLoadExecutor-3903-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/./collection1/data/]
   [junit4]   2> 1856854 INFO  (coreLoadExecutor-3903-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=17, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15127797993892625]
   [junit4]   2> 1856899 INFO  (coreLoadExecutor-3903-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1856899 INFO  (coreLoadExecutor-3903-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1856900 INFO  (coreLoadExecutor-3903-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=15, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.705818942854619]
   [junit4]   2> 1856900 INFO  (coreLoadExecutor-3903-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1dc1063[collection1] main]
   [junit4]   2> 1856900 INFO  (coreLoadExecutor-3903-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/collection1/conf
   [junit4]   2> 1856901 INFO  (coreLoadExecutor-3903-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1856901 INFO  (searcherExecutor-3904-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@1dc1063[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1856903 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@2f68bc42{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1856903 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1580894958
   [junit4]   2> 1856903 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 1856903 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6c61c044: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@65b76bdf
   [junit4]   2> 1856905 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1856905 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6d0cd708: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@1691e6c9
   [junit4]   2> 1856907 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1856907 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@21112a6e: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@1736b3c5
   [junit4]   2> 1856908 INFO  (coreCloseExecutor-3908-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@7042a832
   [junit4]   2> 1856909 INFO  (coreCloseExecutor-3908-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1883416626
   [junit4]   2> 1856909 INFO  (coreCloseExecutor-3908-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@ddb798f: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@344b0c5c
   [junit4]   2> 1856914 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@746b077f{/solr,null,UNAVAILABLE}
   [junit4]   2> 1856916 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1856917 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7ba1e889{/solr,null,AVAILABLE}
   [junit4]   2> 1856917 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@33ad50a6{HTTP/1.1,[http/1.1]}{127.0.0.1:45000}
   [junit4]   2> 1856917 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.e.j.s.Server Started @1860380ms
   [junit4]   2> 1856917 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-001/collection1/data, hostContext=/solr, hostPort=45000}
   [junit4]   2> 1856917 ERROR (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1856917 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1856918 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on port null
   [junit4]   2> 1856918 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1856918 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-03T22:13:28.149Z
   [junit4]   2> 1856918 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-001/solr.xml
   [junit4]   2> 1856921 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1856951 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc
   [junit4]   2> 1856957 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc
   [junit4]   2> 1856958 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc
   [junit4]   2> 1856959 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-001/.
   [junit4]   2> 1856959 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1856960 INFO  (coreLoadExecutor-3915-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 1856968 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1856973 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1856975 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 1856975 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-001/./collection1, trusted=true
   [junit4]   2> 1856975 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc
   [junit4]   2> 1856975 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1856975 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-001/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-001/./collection1/data/]
   [junit4]   2> 1856976 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=17, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15127797993892625]
   [junit4]   2> 1857023 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1857023 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1857024 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=15, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.705818942854619]
   [junit4]   2> 1857024 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@76acbec1[collection1] main]
   [junit4]   2> 1857024 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-001/collection1/conf
   [junit4]   2> 1857025 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following config files: schema.xml
   [junit4]   2> 1857025 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1857026 INFO  (searcherExecutor-3916-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@76acbec1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1857028 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 1857029 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{deleteByQuery=*:*} 0 0
   [junit4]   2> 1857031 INFO  (qtp420576524-12655) [    x:collection1] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 1857031 INFO  (qtp420576524-12655) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{deleteByQuery=*:*} 0 0
   [junit4]   2> 1857032 INFO  (qtp420576524-12656) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1857032 INFO  (qtp420576524-12656) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@3dd5cdd8 commitCommandVersion:0
   [junit4]   2> 1857035 INFO  (qtp420576524-12656) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@372da6a8[collection1] main]
   [junit4]   2> 1857036 INFO  (qtp420576524-12656) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1857036 INFO  (searcherExecutor-3904-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@372da6a8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1857037 INFO  (qtp420576524-12656) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 4
   [junit4]   2> 1857038 INFO  (qtp1138952998-12675) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[0]} 0 0
   [junit4]   2> 1857039 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[1]} 0 0
   [junit4]   2> 1857039 INFO  (qtp1138952998-12676) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[2]} 0 0
   [junit4]   2> 1857039 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[3]} 0 0
   [junit4]   2> 1857040 INFO  (qtp1138952998-12677) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[4]} 0 0
   [junit4]   2> 1857040 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[5]} 0 0
   [junit4]   2> 1857040 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[6]} 0 0
   [junit4]   2> 1857041 INFO  (qtp1138952998-12675) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[7]} 0 0
   [junit4]   2> 1857041 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[8]} 0 0
   [junit4]   2> 1857041 INFO  (qtp1138952998-12676) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[9]} 0 0
   [junit4]   2> 1857042 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[10]} 0 0
   [junit4]   2> 1857042 INFO  (qtp1138952998-12677) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[11]} 0 0
   [junit4]   2> 1857042 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[12]} 0 0
   [junit4]   2> 1857043 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[13]} 0 0
   [junit4]   2> 1857043 INFO  (qtp1138952998-12675) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[14]} 0 0
   [junit4]   2> 1857044 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[15]} 0 0
   [junit4]   2> 1857044 INFO  (qtp1138952998-12676) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[16]} 0 0
   [junit4]   2> 1857044 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[17]} 0 0
   [junit4]   2> 1857045 INFO  (qtp1138952998-12677) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[18]} 0 0
   [junit4]   2> 1857045 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[19]} 0 0
   [junit4]   2> 1857045 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[20]} 0 0
   [junit4]   2> 1857046 INFO  (qtp1138952998-12675) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[21]} 0 0
   [junit4]   2> 1857046 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[22]} 0 0
   [junit4]   2> 1857046 INFO  (qtp1138952998-12676) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[23]} 0 0
   [junit4]   2> 1857047 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[24]} 0 0
   [junit4]   2> 1857047 INFO  (qtp1138952998-12677) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[25]} 0 0
   [junit4]   2> 1857047 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[26]} 0 0
   [junit4]   2> 1857048 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[27]} 0 0
   [junit4]   2> 1857048 INFO  (qtp1138952998-12675) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[28]} 0 0
   [junit4]   2> 1857048 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[29]} 0 0
   [junit4]   2> 1857049 INFO  (qtp1138952998-12676) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[30]} 0 0
   [junit4]   2> 1857049 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[31]} 0 0
   [junit4]   2> 1857049 INFO  (qtp1138952998-12677) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[32]} 0 0
   [junit4]   2> 1857050 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[33]} 0 0
   [junit4]   2> 1857050 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[34]} 0 0
   [junit4]   2> 1857050 INFO  (qtp1138952998-12675) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[35]} 0 0
   [junit4]   2> 1857051 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[36]} 0 0
   [junit4]   2> 1857051 INFO  (qtp1138952998-12676) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[37]} 0 0
   [junit4]   2> 1857051 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[38]} 0 0
   [junit4]   2> 1857051 INFO  (qtp1138952998-12677) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[39]} 0 0
   [junit4]   2> 1857052 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[40]} 0 0
   [junit4]   2> 1857052 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[41]} 0 0
   [junit4]   2> 1857052 INFO  (qtp1138952998-12675) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[42]} 0 0
   [junit4]   2> 1857053 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[43]} 0 0
   [junit4]   2> 1857053 INFO  (qtp1138952998-12676) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[44]} 0 0
   [junit4]   2> 1857053 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[45]} 0 0
   [junit4]   2> 1857053 INFO  (qtp1138952998-12677) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[46]} 0 0
   [junit4]   2> 1857054 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[47]} 0 0
   [junit4]   2> 1857054 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[48]} 0 0
   [junit4]   2> 1857054 INFO  (qtp1138952998-12675) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[49]} 0 0
   [junit4]   2> 1857054 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[50]} 0 0
   [junit4]   2> 1857055 INFO  (qtp1138952998-12676) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[51]} 0 0
   [junit4]   2> 1857055 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[52]} 0 0
   [junit4]   2> 1857055 INFO  (qtp1138952998-12677) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[53]} 0 0
   [junit4]   2> 1857055 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[54]} 0 0
   [junit4]   2> 1857056 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[55]} 0 0
   [junit4]   2> 1857056 INFO  (qtp1138952998-12675) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[56]} 0 0
   [junit4]   2> 1857056 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[57]} 0 0
   [junit4]   2> 1857056 INFO  (qtp1138952998-12676) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[58]} 0 0
   [junit4]   2> 1857057 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[59]} 0 0
   [junit4]   2> 1857057 INFO  (qtp1138952998-12677) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[60]} 0 0
   [junit4]   2> 1857057 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[61]} 0 0
   [junit4]   2> 1857058 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[62]} 0 0
   [junit4]   2> 1857058 INFO  (qtp1138952998-12675) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[63]} 0 0
   [junit4]   2> 1857058 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[64]} 0 0
   [junit4]   2> 1857059 INFO  (qtp1138952998-12676) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[65]} 0 0
   [junit4]   2> 1857059 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[66]} 0 0
   [junit4]   2> 1857059 INFO  (qtp1138952998-12677) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[67]} 0 0
   [junit4]   2> 1857059 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[68]} 0 0
   [junit4]   2> 1857060 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[69]} 0 0
   [junit4]   2> 1857060 INFO  (qtp1138952998-12675) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[70]} 0 0
   [junit4]   2> 1857061 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[71]} 0 0
   [junit4]   2> 1857061 INFO  (qtp1138952998-12676) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[72]} 0 0
   [junit4]   2> 1857061 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[73]} 0 0
   [junit4]   2> 1857061 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[74]} 0 0
   [junit4]   2> 1857062 INFO  (qtp1138952998-12677) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[75]} 0 0
   [junit4]   2> 1857062 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[76]} 0 0
   [junit4]   2> 1857063 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[77]} 0 0
   [junit4]   2> 1857063 INFO  (qtp1138952998-12675) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[78]} 0 0
   [junit4]   2> 1857063 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[79]} 0 0
   [junit4]   2> 1857064 INFO  (qtp1138952998-12676) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[80]} 0 0
   [junit4]   2> 1857064 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[81]} 0 0
   [junit4]   2> 1857064 INFO  (qtp1138952998-12677) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[82]} 0 0
   [junit4]   2> 1857065 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[83]} 0 0
   [junit4]   2> 1857065 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[84]} 0 0
   [junit4]   2> 1857065 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[85]} 0 0
   [junit4]   2> 1857066 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[86]} 0 0
   [junit4]   2> 1857066 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[87]} 0 0
   [junit4]   2> 1857066 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[88]} 0 0
   [junit4]   2> 1857067 INFO  (qtp1138952998-12675) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[89]} 0 0
   [junit4]   2> 1857067 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[90]} 0 0
   [junit4]   2> 1857067 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[91]} 0 0
   [junit4]   2> 1857067 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[92]} 0 0
   [junit4]   2> 1857068 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[93]} 0 0
   [junit4]   2> 1857068 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[94]} 0 0
   [junit4]   2> 1857068 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[95]} 0 0
   [junit4]   2> 1857068 INFO  (qtp1138952998-12677) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[96]} 0 0
   [junit4]   2> 1857068 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[97]} 0 0
   [junit4]   2> 1857069 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[98]} 0 0
   [junit4]   2> 1857069 INFO  (qtp1138952998-12675) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[99]} 0 0
   [junit4]   2> 1857069 INFO  (qtp1138952998-12676) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[100]} 0 0
   [junit4]   2> 1857070 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[101]} 0 0
   [junit4]   2> 1857070 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[102]} 0 0
   [junit4]   2> 1857070 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[103]} 0 0
   [junit4]   2> 1857071 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[104]} 0 0
   [junit4]   2> 1857071 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[105]} 0 0
   [junit4]   2> 1857071 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[106]} 0 0
   [junit4]   2> 1857071 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[107]} 0 0
   [junit4]   2> 1857071 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[108]} 0 0
   [junit4]   2> 1857073 INFO  (qtp1138952998-12675) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[109]} 0 0
   [junit4]   2> 1857073 INFO  (qtp1138952998-12676) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[110]} 0 0
   [junit4]   2> 1857073 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[111]} 0 0
   [junit4]   2> 1857074 INFO  (qtp1138952998-12677) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[112]} 0 0
   [junit4]   2> 1857074 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[113]} 0 0
   [junit4]   2> 1857074 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[114]} 0 0
   [junit4]   2> 1857075 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[115]} 0 0
   [junit4]   2> 1857075 INFO  (qtp1138952998-12675) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[116]} 0 0
   [junit4]   2> 1857075 INFO  (qtp1138952998-12676) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[117]} 0 0
   [junit4]   2> 1857076 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[118]} 0 0
   [junit4]   2> 1857076 INFO  (qtp1138952998-12677) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[119]} 0 0
   [junit4]   2> 1857076 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[120]} 0 0
   [junit4]   2> 1857077 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[121]} 0 0
   [junit4]   2> 1857077 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[122]} 0 0
   [junit4]   2> 1857077 INFO  (qtp1138952998-12675) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[123]} 0 0
   [junit4]   2> 1857078 INFO  (qtp1138952998-12676) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[124]} 0 0
   [junit4]   2> 1857078 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[125]} 0 0
   [junit4]   2> 1857078 INFO  (qtp1138952998-12677) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[126]} 0 0
   [junit4]   2> 1857079 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[127]} 0 0
   [junit4]   2> 1857079 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[128]} 0 0
   [junit4]   2> 1857080 INFO  (qtp1138952998-12678) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[129]} 0 0
   [junit4]   2> 1857080 INFO  (qtp1138952998-12675) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[130]} 0 0
   [junit4]   2> 1857080 INFO  (qtp1138952998-12676) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[131]} 0 0
   [junit4]   2> 1857081 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1857081 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@29d4b274 commitCommandVersion:0
   [junit4]   2> 1857088 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6fec4eeb[collection1] main]
   [junit4]   2> 1857089 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1857089 INFO  (searcherExecutor-3916-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6fec4eeb[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):c132)))}
   [junit4]   2> 1857090 INFO  (qtp1138952998-12672) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=/solr path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 8
   [junit4]   2> 1857094 INFO  (qtp1138952998-12677) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/select params={q=*:*&sort=id+desc&wt=javabin&version=2} hits=132 status=0 QTime=3
   [junit4]   2> 1857094 INFO  (TEST-TestReplicationHandler.doTestStressReplication-seed#[7C1E330DBD08839F]) [    ] o.a.s.h.TestReplicationHandler Waited for 0ms and found 132 docs
   [junit4]   2> 1857096 INFO  (qtp420576524-12658) [    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={core=collection1&action=status&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1857100 WARN  (explicit-fetchindex-cmd) [    x:collection1] o.a.s.h.IndexFetcher 'masterUrl' must be specified without the /replication suffix
   [junit4]   2> 1857101 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 1857102 INFO  (explicit-fetchindex-cmd) [    x:collection1] o.a.s.h.IndexFetcher Master's generation: 2
   [junit4]   2> 1857102 INFO  (explicit-fetchindex-cmd) [    x:collection1] o.a.s.h.IndexFetcher Master's version: 1507068808312
   [junit4]   2> 1857102 INFO  (explicit-fetchindex-cmd) [    x:collection1] o.a.s.h.IndexFetcher Slave's generation: 2
   [junit4]   2> 1857102 INFO  (explicit-fetchindex-cmd) [    x:collection1] o.a.s.h.IndexFetcher Slave's version: 1507068808263
   [junit4]   2> 1857102 INFO  (explicit-fetchindex-cmd) [    x:collection1] o.a.s.h.IndexFetcher Starting replication process
   [junit4]   2> 1857104 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=2&qt=/replication&wt=javabin&version=2&command=filelist} status=0 QTime=2
   [junit4]   2> 1857104 INFO  (explicit-fetchindex-cmd) [    x:collection1] o.a.s.h.IndexFetcher Number of files in latest index in master: 4
   [junit4]   2> 1857105 INFO  (explicit-fetchindex-cmd) [    x:collection1] o.a.s.h.IndexFetcher Starting download (fullCopy=true) to NRTCachingDirectory(MMapDirectory@/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/collection1/data/index.20171003171328336 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2b3f050a; maxCacheMB=48.0 maxMergeSizeMB=4.0)
   [junit4]   2> 1857105 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.cfe&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 1857107 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.si&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 1857107 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=_0.cfs&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 1857108 WARN  (explicit-fetchindex-cmd) [    x:collection1] o.a.s.h.IndexFetcher File segments_2 did not match. expected checksum is 2177212637 and actual is checksum 2357558756. expected length is 185 and actual length is 120
   [junit4]   2> 1857109 INFO  (qtp1138952998-12671) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=2&qt=/replication&file=segments_2&checksum=true&wt=filestream&command=filecontent} status=0 QTime=0
   [junit4]   2> 1857109 INFO  (explicit-fetchindex-cmd) [    x:collection1] o.a.s.h.IndexFetcher Total time taken for download (fullCopy=true,bytesDownloaded=5386) : 0 secs (null bytes/sec) to NRTCachingDirectory(MMapDirectory@/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/collection1/data/index.20171003171328336 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2b3f050a; maxCacheMB=48.0 maxMergeSizeMB=4.0)
   [junit4]   2> 1857109 INFO  (explicit-fetchindex-cmd) [    x:collection1] o.a.s.h.IndexFetcher Starting download of configuration files from master: [{size=1530, name=schema.xml, checksum=1508380349}]
   [junit4]   2> 1857119 INFO  (qtp1138952998-12674) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={generation=2&qt=/replication&cf=schema.xml&checksum=true&wt=filestream&command=filecontent} status=0 QTime=4
   [junit4]   2> 1857122 INFO  (explicit-fetchindex-cmd) [    x:collection1] o.a.s.c.SolrCore Updating index properties... index=index.20171003171328336
   [junit4]   2> 1857123 INFO  (explicit-fetchindex-cmd) [    x:collection1] o.a.s.h.IndexFetcher Configuration files are modified, core will be reloaded
   [junit4]   2> 1857124 INFO  (explicit-fetchindex-cmd) [    x:collection1] o.a.s.h.IndexFetcher Reloading SolrCore collection1
   [junit4]   2> 1857132 INFO  (Thread-4499) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1857137 INFO  (Thread-4499) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1857138 INFO  (Thread-4499) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 1857138 INFO  (Thread-4499) [    x:collection1] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/./collection1
   [junit4]   2> 1857155 INFO  (Thread-4499) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c829ccc
   [junit4]   2> 1857155 INFO  (Thread-4499) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/./collection1/data/]
   [junit4]   2> 1857157 INFO  (OldIndexDirectoryCleanupThreadForCore-collection1) [    x:collection1] o.a.s.c.DirectoryFactory Will not remove most recent old directory after reload /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/./collection1/data/index
   [junit4]   2> 1857157 INFO  (OldIndexDirectoryCleanupThreadForCore-collection1) [    x:collection1] o.a.s.c.DirectoryFactory Found 0 old index directories to clean-up under /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/./collection1/data/ afterReload=true
   [junit4]   2> 1857197 INFO  (Thread-4499) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1857197 INFO  (Thread-4499) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1857198 INFO  (Thread-4499) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@744e6c67[collection1] main]
   [junit4]   2> 1857198 INFO  (Thread-4499) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_7C1E330DBD08839F-001/solr-instance-002/collection1/conf
   [junit4]   2> 1857198 INFO  (Thread-4499) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1857199 INFO  (Thread-4499) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=17, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15127797993892625]
   [junit4]   2> 1857200 INFO  (searcherExecutor-3921-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@744e6c67[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1857200 INFO  (Thread-4499) [    x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 1857200 INFO  (Thread-4499) [    x:collection1] o.a.s.s.SolrIndexSearcher 

[...truncated too long message...]

41928-127.0.0.1:36452_solr-n_0000000000) closing
   [junit4]   2> 213896 INFO  (OverseerStateUpdate-98767455426641928-127.0.0.1:36452_solr-n_0000000000) [n:127.0.0.1:36452_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:36452_solr
   [junit4]   2> 213903 INFO  (zkCallback-337-thread-1-processing-n:127.0.0.1:50059_solr) [n:127.0.0.1:50059_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:50059_solr
   [junit4]   2> 213906 INFO  (zkCallback-337-thread-3-processing-n:127.0.0.1:50059_solr) [n:127.0.0.1:50059_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 213912 INFO  (jetty-closer-316-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@16d8b2b8{/solr,null,UNAVAILABLE}
   [junit4]   2> 215358 WARN  (zkCallback-336-thread-4-processing-n:127.0.0.1:33029_solr) [n:127.0.0.1:33029_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 215358 INFO  (jetty-closer-316-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@30c50ffa{/solr,null,UNAVAILABLE}
   [junit4]   2> 215419 WARN  (zkCallback-337-thread-1-processing-n:127.0.0.1:50059_solr) [n:127.0.0.1:50059_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 215419 INFO  (jetty-closer-316-thread-4) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5a7bb3d5{/solr,null,UNAVAILABLE}
   [junit4]   2> 215421 ERROR (SUITE-StreamExpressionTest-seed#[EAD91D9727B41A1]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 215421 INFO  (SUITE-StreamExpressionTest-seed#[EAD91D9727B41A1]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39171 39171
   [junit4]   2> 225465 INFO  (Thread-219) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39171 39171
   [junit4]   2> 225467 WARN  (Thread-219) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	78	/solr/clusterprops.json
   [junit4]   2> 	78	/solr/aliases.json
   [junit4]   2> 	8	/solr/configs/conf
   [junit4]   2> 	4	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	78	/solr/clusterstate.json
   [junit4]   2> 	22	/solr/collections/destination/state.json
   [junit4]   2> 	22	/solr/collections/mainCorpus/state.json
   [junit4]   2> 	20	/solr/collections/parallelDestinationCollection/state.json
   [junit4]   2> 	20	/solr/collections/workQueue/state.json
   [junit4]   2> 	20	/solr/collections/destinationCollection/state.json
   [junit4]   2> 	18	/solr/collections/parallelDestinationCollection1/state.json
   [junit4]   2> 	12	/solr/collections/collection1_collection/state.json
   [junit4]   2> 	2	/solr/overseer_elect/election/98767455426641931-127.0.0.1:33029_solr-n_0000000001
   [junit4]   2> 	2	/solr/overseer_elect/election/98767455426641928-127.0.0.1:36452_solr-n_0000000000
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	78	/solr/live_nodes
   [junit4]   2> 	78	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.io.stream.StreamExpressionTest_EAD91D9727B41A1-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {expr_s=PostingsFormat(name=Memory), join2_s=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), multiDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), subject=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), price_f=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), a1_s=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), intDefault=FST50, a_f=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), a2_s=Lucene50(blocksize=128), join1_i=PostingsFormat(name=Memory), a_i=PostingsFormat(name=Memory), test_i=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), join3_i=Lucene50(blocksize=128), field_i=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), level1_s=PostingsFormat(name=Memory), body_t=Lucene50(blocksize=128), s_multi=Lucene50(blocksize=128), col_s=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), level2_s=FST50, a_ss=FST50, a_s=Lucene50(blocksize=128), a_t=FST50, id=PostingsFormat(name=Memory), text=Lucene50(blocksize=128), timestamp=FST50, i_multi=PostingsFormat(name=Memory), b_ls=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), test_dt=PostingsFormat(name=Memory), test_t=Lucene50(blocksize=128), ident_s=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), order_i=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), _version_=FST50, side_s=Lucene50(blocksize=128), miles_i=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{expr_s=DocValuesFormat(name=Lucene70), join2_s=DocValuesFormat(name=Lucene70), multiDefault=DocValuesFormat(name=Lucene70), price_f=DocValuesFormat(name=Lucene70), a1_s=DocValuesFormat(name=Lucene70), intDefault=DocValuesFormat(name=Memory), a_f=DocValuesFormat(name=Lucene70), a2_s=DocValuesFormat(name=Direct), join1_i=DocValuesFormat(name=Lucene70), a_i=DocValuesFormat(name=Lucene70), test_i=DocValuesFormat(name=Lucene70), join3_i=DocValuesFormat(name=Direct), field_i=DocValuesFormat(name=Lucene70), level1_s=DocValuesFormat(name=Lucene70), s_multi=DocValuesFormat(name=Direct), col_s=DocValuesFormat(name=Lucene70), level2_s=DocValuesFormat(name=Memory), a_ss=DocValuesFormat(name=Memory), a_s=DocValuesFormat(name=Direct), id=DocValuesFormat(name=Lucene70), timestamp=DocValuesFormat(name=Memory), i_multi=DocValuesFormat(name=Lucene70), b_ls=DocValuesFormat(name=Lucene70), test_dt=DocValuesFormat(name=Lucene70), ident_s=DocValuesFormat(name=Lucene70), order_i=DocValuesFormat(name=Lucene70), _version_=DocValuesFormat(name=Memory), side_s=DocValuesFormat(name=Direct), miles_i=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=841, maxMBSortInHeap=7.019418034104843, sim=RandomSimilarity(queryNorm=true): {}, locale=de-AT, timezone=America/Creston
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=102801760,total=427646976
   [junit4]   2> NOTE: All tests run in this JVM: [JsonValidatorTest, TestClusteringResponse, TestSolrProperties, TestUpdateRequestCodec, HttpSolrClientConPoolTest, CollectionAdminRequestRequiredParamsTest, TangentEvaluatorTest, SolrDocumentTest, CoalesceEvaluatorTest, ArcTangentEvaluatorTest, TestSuggesterResponse, SolrExampleStreamingTest, AndEvaluatorTest, TestXMLEscaping, RoundEvaluatorTest, SolrPingTest, FieldValueEvaluatorTest, SolrSchemalessExampleTest, FieldAnalysisResponseTest, ConcurrentUpdateSolrClientTest, StreamExpressionParserTest, DivideEvaluatorTest, JdbcDriverTest, TestConfigSetAdminRequest, EqualToEvaluatorTest, ModifiableSolrParamsTest, TestCoreAdmin, SolrExampleXMLTest, TestPathTrie, CorrelationEvaluatorTest, OrEvaluatorTest, GraphExpressionTest, LBHttpSolrClientBuilderTest, JDBCStreamTest, HyperbolicCosineEvaluatorTest, JettyWebappTest, SolrExampleEmbeddedTest, TestBatchUpdate, LargeVolumeJettyTest, SolrParamTest, TermsResponseTest, TestEmbeddedSolrServer, TestJavaBinCodec, TestHash, TestFastInputStream, NamedListTest, FacetFieldTest, BasicHttpSolrClientTest, CloudSolrClientBuilderTest, CloudSolrClientCacheTest, CloudSolrClientTest, StreamExpressionTest]
   [junit4] Completed [137/145 (1!)] on J0 in 88.06s, 117 tests, 1 error, 9 skipped <<< FAILURES!

[...truncated 45540 lines...]
-documentation-lint:
    [jtidy] Checking for broken html (such as invalid tags)...
   [delete] Deleting directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/build/jtidy_tmp
     [echo] Checking for broken links...
     [exec] 
     [exec] Crawl/parse...
     [exec] 
     [exec] Verify...
     [exec] 
     [exec] file:///export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/docs/quickstart.html
     [exec]   BAD EXTERNAL LINK: https://lucene.apache.org/solr/guide/solr-tutorial.html
     [exec] 
     [exec] Broken javadocs links were found! Common root causes:
     [exec] * A typo of some sort for manually created links.
     [exec] * Public methods referencing non-public classes in their signature.

BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:826: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:101: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build.xml:669: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build.xml:682: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/common-build.xml:2570: exec returned: 1

Total time: 110 minutes 54 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1451 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1451/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseG1GC

All tests passed

Build Log:
[...truncated 60276 lines...]
-documentation-lint:
    [jtidy] Checking for broken html (such as invalid tags)...
   [delete] Deleting directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/build/jtidy_tmp
     [echo] Checking for broken links...
     [exec] 
     [exec] Crawl/parse...
     [exec] 
     [exec] Verify...
     [exec] 
     [exec] file:///export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/docs/quickstart.html
     [exec]   BAD EXTERNAL LINK: https://lucene.apache.org/solr/guide/solr-tutorial.html
     [exec] 
     [exec] Broken javadocs links were found! Common root causes:
     [exec] * A typo of some sort for manually created links.
     [exec] * Public methods referencing non-public classes in their signature.

BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:826: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:101: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build.xml:669: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build.xml:682: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/common-build.xml:2570: exec returned: 1

Total time: 100 minutes 52 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1450 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1450/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

All tests passed

Build Log:
[...truncated 60306 lines...]
-documentation-lint:
    [jtidy] Checking for broken html (such as invalid tags)...
   [delete] Deleting directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/build/jtidy_tmp
     [echo] Checking for broken links...
     [exec] 
     [exec] Crawl/parse...
     [exec] 
     [exec] Verify...
     [exec] 
     [exec] file:///export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/docs/quickstart.html
     [exec]   BAD EXTERNAL LINK: https://lucene.apache.org/solr/guide/solr-tutorial.html
     [exec] 
     [exec] Broken javadocs links were found! Common root causes:
     [exec] * A typo of some sort for manually created links.
     [exec] * Public methods referencing non-public classes in their signature.

BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:826: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:101: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build.xml:669: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build.xml:682: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/common-build.xml:2570: exec returned: 1

Total time: 96 minutes 53 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1449 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1449/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:     1) Thread[id=30319, name=jetty-launcher-4309-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   1) Thread[id=30319, name=jetty-launcher-4309-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
	at __randomizedtesting.SeedInfo.seed([8EBC4DF65222051A]:0)




Build Log:
[...truncated 13266 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_8EBC4DF65222051A-001/init-core-data-001
   [junit4]   2> 3399209 WARN  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[8EBC4DF65222051A]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4 numCloses=4
   [junit4]   2> 3399209 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[8EBC4DF65222051A]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 3399211 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[8EBC4DF65222051A]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 3399211 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[8EBC4DF65222051A]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 3399251 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[8EBC4DF65222051A]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_8EBC4DF65222051A-001/tempDir-001
   [junit4]   2> 3399251 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[8EBC4DF65222051A]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 3399252 INFO  (Thread-8338) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 3399252 INFO  (Thread-8338) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 3399255 ERROR (Thread-8338) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 3399353 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[8EBC4DF65222051A]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:54040
   [junit4]   2> 3399372 INFO  (jetty-launcher-4309-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 3399372 INFO  (jetty-launcher-4309-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 3399374 INFO  (jetty-launcher-4309-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@22999a72{/solr,null,AVAILABLE}
   [junit4]   2> 3399374 INFO  (jetty-launcher-4309-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@285c447d{/solr,null,AVAILABLE}
   [junit4]   2> 3399375 INFO  (jetty-launcher-4309-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@ad947b4{SSL,[ssl, http/1.1]}{127.0.0.1:39694}
   [junit4]   2> 3399375 INFO  (jetty-launcher-4309-thread-1) [    ] o.e.j.s.Server Started @3403109ms
   [junit4]   2> 3399375 INFO  (jetty-launcher-4309-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39694}
   [junit4]   2> 3399375 ERROR (jetty-launcher-4309-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3399375 INFO  (jetty-launcher-4309-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@79c82588{SSL,[ssl, http/1.1]}{127.0.0.1:61472}
   [junit4]   2> 3399376 INFO  (jetty-launcher-4309-thread-2) [    ] o.e.j.s.Server Started @3403110ms
   [junit4]   2> 3399376 INFO  (jetty-launcher-4309-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 3399376 INFO  (jetty-launcher-4309-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3399376 INFO  (jetty-launcher-4309-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=61472}
   [junit4]   2> 3399376 INFO  (jetty-launcher-4309-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 3399376 INFO  (jetty-launcher-4309-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-02T13:40:05.658Z
   [junit4]   2> 3399376 ERROR (jetty-launcher-4309-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3399376 INFO  (jetty-launcher-4309-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 3399376 INFO  (jetty-launcher-4309-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3399376 INFO  (jetty-launcher-4309-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 3399376 INFO  (jetty-launcher-4309-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-02T13:40:05.658Z
   [junit4]   2> 3399378 INFO  (jetty-launcher-4309-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3399382 INFO  (jetty-launcher-4309-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3399390 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15edd50ad1f0002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 3399394 INFO  (jetty-launcher-4309-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54040/solr
   [junit4]   2> 3399396 INFO  (jetty-launcher-4309-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54040/solr
   [junit4]   2> 3399438 INFO  (jetty-launcher-4309-thread-1) [n:127.0.0.1:39694_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 3399439 INFO  (jetty-launcher-4309-thread-1) [n:127.0.0.1:39694_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:39694_solr
   [junit4]   2> 3399440 INFO  (jetty-launcher-4309-thread-1) [n:127.0.0.1:39694_solr    ] o.a.s.c.Overseer Overseer (id=98759580420341766-127.0.0.1:39694_solr-n_0000000000) starting
   [junit4]   2> 3399455 INFO  (jetty-launcher-4309-thread-1) [n:127.0.0.1:39694_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39694_solr
   [junit4]   2> 3399457 INFO  (zkCallback-4321-thread-1-processing-n:127.0.0.1:39694_solr) [n:127.0.0.1:39694_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3399594 INFO  (jetty-launcher-4309-thread-2) [n:127.0.0.1:61472_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3399597 INFO  (jetty-launcher-4309-thread-2) [n:127.0.0.1:61472_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 3399599 INFO  (jetty-launcher-4309-thread-2) [n:127.0.0.1:61472_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:61472_solr
   [junit4]   2> 3399601 INFO  (zkCallback-4320-thread-1-processing-n:127.0.0.1:61472_solr) [n:127.0.0.1:61472_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3399601 INFO  (zkCallback-4321-thread-1-processing-n:127.0.0.1:39694_solr) [n:127.0.0.1:39694_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3399630 INFO  (jetty-launcher-4309-thread-1) [n:127.0.0.1:39694_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 3399631 WARN  (jetty-launcher-4309-thread-1) [n:127.0.0.1:39694_solr    ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
   [junit4]   2> 3399634 INFO  (jetty-launcher-4309-thread-1) [n:127.0.0.1:39694_solr    ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
   [junit4]   2> 3399682 INFO  (jetty-launcher-4309-thread-2) [n:127.0.0.1:61472_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 3399683 WARN  (jetty-launcher-4309-thread-2) [n:127.0.0.1:61472_solr    ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
   [junit4]   2> 3399683 INFO  (jetty-launcher-4309-thread-2) [n:127.0.0.1:61472_solr    ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
   [junit4]   2> 3399726 INFO  (jetty-launcher-4309-thread-1) [n:127.0.0.1:39694_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 3399726 INFO  (jetty-launcher-4309-thread-2) [n:127.0.0.1:61472_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 3399767 INFO  (jetty-launcher-4309-thread-2-EventThread) [n:127.0.0.1:61472_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 3399768 INFO  (jetty-launcher-4309-thread-1-EventThread) [n:127.0.0.1:39694_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 3399938 WARN  (jetty-launcher-4309-thread-2) [n:127.0.0.1:61472_solr    ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
   [junit4]   2> 3399943 WARN  (jetty-launcher-4309-thread-1) [n:127.0.0.1:39694_solr    ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
   [junit4]   2> 3400007 INFO  (jetty-launcher-4309-thread-2) [n:127.0.0.1:61472_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_61472.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ef8a2f6
   [junit4]   2> 3400011 INFO  (jetty-launcher-4309-thread-1) [n:127.0.0.1:39694_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39694.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ef8a2f6
   [junit4]   2> 3400013 INFO  (jetty-launcher-4309-thread-2) [n:127.0.0.1:61472_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_61472.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ef8a2f6
   [junit4]   2> 3400013 INFO  (jetty-launcher-4309-thread-2) [n:127.0.0.1:61472_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_61472.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ef8a2f6
   [junit4]   2> 3400014 INFO  (jetty-launcher-4309-thread-2) [n:127.0.0.1:61472_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_8EBC4DF65222051A-001/tempDir-001/node2/.
   [junit4]   2> 3400024 INFO  (jetty-launcher-4309-thread-1) [n:127.0.0.1:39694_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39694.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ef8a2f6
   [junit4]   2> 3400024 INFO  (jetty-launcher-4309-thread-1) [n:127.0.0.1:39694_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39694.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ef8a2f6
   [junit4]   2> 3400026 INFO  (jetty-launcher-4309-thread-1) [n:127.0.0.1:39694_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_8EBC4DF65222051A-001/tempDir-001/node1/.
   [junit4]   2> 3400212 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateAnyHostAnyUser
   [junit4]   2> 3400258 INFO  (qtp444197900-30280) [n:127.0.0.1:61472_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3400258 INFO  (qtp444197900-30280) [n:127.0.0.1:61472_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3400259 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateAnyHostAnyUser
   [junit4]   2> 3400442 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateGroup
   [junit4]   2> 3400486 INFO  (qtp444197900-30274) [n:127.0.0.1:61472_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=jenkins&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3400486 INFO  (qtp444197900-30274) [n:127.0.0.1:61472_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=jenkins&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3400486 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateGroup
   [junit4]   2> 3400643 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigGroups
   [junit4]   2> 3400660 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigGroups
   [junit4]   2> 3400867 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testForwarding
   [junit4]   2> 3401473 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[8EBC4DF65222051A]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3401475 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[8EBC4DF65222051A]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:54040/solr ready
   [junit4]   2> 3401510 INFO  (qtp449375795-30287) [n:127.0.0.1:39694_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3401514 INFO  (OverseerThreadFactory-11650-thread-1-processing-n:127.0.0.1:39694_solr) [n:127.0.0.1:39694_solr    ] o.a.s.c.CreateCollectionCmd Create collection forwardingCollection
   [junit4]   2> 3401626 INFO  (OverseerStateUpdate-98759580420341766-127.0.0.1:39694_solr-n_0000000000) [n:127.0.0.1:39694_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"forwardingCollection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"forwardingCollection_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:61472/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 3401839 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica_n1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 3401840 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 3401947 INFO  (zkCallback-4320-thread-1-processing-n:127.0.0.1:61472_solr) [n:127.0.0.1:61472_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 3401947 INFO  (zkCallback-4320-thread-2-processing-n:127.0.0.1:61472_solr) [n:127.0.0.1:61472_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 3402867 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 3402900 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.IndexSchema [forwardingCollection_shard1_replica_n1] Schema name=test
   [junit4]   2> 3403031 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 3403058 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'forwardingCollection_shard1_replica_n1' using configuration from collection forwardingCollection, trusted=true
   [junit4]   2> 3403059 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_61472.solr.core.forwardingCollection.shard1.replica_n1' (registry 'solr.core.forwardingCollection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ef8a2f6
   [junit4]   2> 3403059 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 3403059 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [[forwardingCollection_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_8EBC4DF65222051A-001/tempDir-001/node2/forwardingCollection_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_8EBC4DF65222051A-001/tempDir-001/node2/./forwardingCollection_shard1_replica_n1/data/]
   [junit4]   2> 3403062 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=42, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4927537390059892]
   [junit4]   2> 3403145 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 3403145 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 3403147 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 3403147 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 3403150 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=888736960, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 3403151 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4b7f7012[forwardingCollection_shard1_replica_n1] main]
   [junit4]   2> 3403153 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 3403157 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 3403175 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 3403176 INFO  (searcherExecutor-11655-thread-1-processing-n:127.0.0.1:61472_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 3403176 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580153290839031808
   [junit4]   2> 3403183 INFO  (searcherExecutor-11655-thread-1-processing-n:127.0.0.1:61472_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 3403184 INFO  (searcherExecutor-11655-thread-1-processing-n:127.0.0.1:61472_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 3403184 INFO  (searcherExecutor-11655-thread-1-processing-n:127.0.0.1:61472_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 3403186 INFO  (searcherExecutor-11655-thread-1-processing-n:127.0.0.1:61472_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 3403187 INFO  (searcherExecutor-11655-thread-1-processing-n:127.0.0.1:61472_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 3403188 INFO  (searcherExecutor-11655-thread-1-processing-n:127.0.0.1:61472_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 3403189 INFO  (searcherExecutor-11655-thread-1-processing-n:127.0.0.1:61472_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 3403190 INFO  (searcherExecutor-11655-thread-1-processing-n:127.0.0.1:61472_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 3403192 INFO  (searcherExecutor-11655-thread-1-processing-n:127.0.0.1:61472_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica_n1] Registered new searcher Searcher@4b7f7012[forwardingCollection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 3403194 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 3403195 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 3403195 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:61472/solr/forwardingCollection_shard1_replica_n1/
   [junit4]   2> 3403195 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 3403195 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:61472/solr/forwardingCollection_shard1_replica_n1/ has no replicas
   [junit4]   2> 3403196 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 3403203 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:61472/solr/forwardingCollection_shard1_replica_n1/ shard1
   [junit4]   2> 3403307 INFO  (zkCallback-4320-thread-1-processing-n:127.0.0.1:61472_solr) [n:127.0.0.1:61472_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 3403307 INFO  (zkCallback-4320-thread-2-processing-n:127.0.0.1:61472_solr) [n:127.0.0.1:61472_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 3403355 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 3403358 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica_n1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1519
   [junit4]   2> 3403367 INFO  (qtp449375795-30287) [n:127.0.0.1:39694_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 3403460 INFO  (zkCallback-4320-thread-2-processing-n:127.0.0.1:61472_solr) [n:127.0.0.1:61472_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 3403460 INFO  (zkCallback-4320-thread-1-processing-n:127.0.0.1:61472_solr) [n:127.0.0.1:61472_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 3403519 INFO  (OverseerCollectionConfigSetProcessor-98759580420341766-127.0.0.1:39694_solr-n_0000000000) [n:127.0.0.1:39694_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 3404370 INFO  (qtp449375795-30287) [n:127.0.0.1:39694_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2} status=0 QTime=2860
   [junit4]   2> 3404370 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[8EBC4DF65222051A]) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: forwardingCollection failOnTimeout:true timeout (sec):100
   [junit4]   2> 3404371 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[8EBC4DF65222051A]) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: forwardingCollection
   [junit4]   2> 3404376 INFO  (qtp444197900-30274) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.S.Request [forwardingCollection_shard1_replica_n1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 3404389 INFO  (qtp449375795-30289) [n:127.0.0.1:39694_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 3404393 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.S.Request [forwardingCollection_shard1_replica_n1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 3404394 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testForwarding
   [junit4]   2> 3404584 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyWrongHost
   [junit4]   2> 3409665 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyWrongHost
   [junit4]   2> 3409907 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateHost
   [junit4]   2> 3409909 INFO  (qtp444197900-30280) [n:127.0.0.1:61472_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3409909 INFO  (qtp444197900-30280) [n:127.0.0.1:61472_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3409910 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateHost
   [junit4]   2> 3410171 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigHosts
   [junit4]   2> 3410174 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigHosts
   [junit4]   2> 3410441 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidProxyUser
   [junit4]   2> 3410446 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidProxyUser
   [junit4]   2> 3410725 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidGroup
   [junit4]   2> 3410767 WARN  (qtp444197900-30279) [    ] o.a.h.s.ShellBasedUnixGroupsMapping got exception trying to get groups for user bar: id: invalid user name: "bar"
   [junit4]   2> id: invalid user name: "bar"
   [junit4]   2> 
   [junit4]   2> 3410781 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidGroup
   [junit4]   2> 3411028 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyUnknownRemote
   [junit4]   2> 3427802 WARN  (jetty-launcher-4309-thread-2-SendThread(127.0.0.1:54040)) [n:127.0.0.1:61472_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 17673ms for sessionid 0x15edd50ad1f0004
   [junit4]   2> 3427803 WARN  (jetty-launcher-4309-thread-2-SendThread(127.0.0.1:54040)) [n:127.0.0.1:61472_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 18165ms for sessionid 0x15edd50ad1f0007
   [junit4]   2> 3427803 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15edd50ad1f0004, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 3427802 WARN  (jetty-launcher-4309-thread-1-SendThread(127.0.0.1:54040)) [n:127.0.0.1:39694_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 18157ms for sessionid 0x15edd50ad1f0008
   [junit4]   2> 3427804 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15edd50ad1f0007, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 3427804 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15edd50ad1f0008, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 3427804 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyUnknownRemote
   [junit4]   2> 3427803 WARN  (jetty-launcher-4309-thread-1-SendThread(127.0.0.1:54040)) [n:127.0.0.1:39694_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 18158ms for sessionid 0x15edd50ad1f0006
   [junit4]   2> 3427805 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15edd50ad1f0006, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 3427915 INFO  (jetty-launcher-4309-thread-2-EventThread) [n:127.0.0.1:61472_solr    ] o.a.c.f.s.ConnectionStateManager State change: SUSPENDED
   [junit4]   2> 3427917 WARN  (zkCallback-4320-thread-2-processing-n:127.0.0.1:61472_solr) [n:127.0.0.1:61472_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@50ba4a78 name: ZooKeeperConnection Watcher:127.0.0.1:54040/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 3427927 WARN  (zkCallback-4320-thread-2-processing-n:127.0.0.1:61472_solr) [n:127.0.0.1:61472_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 3427927 WARN  (zkCallback-4328-thread-4) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1aa8cdd3 name: ZooKeeperConnection Watcher:127.0.0.1:54040/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 3427927 WARN  (zkCallback-4328-thread-4) [    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 3427953 WARN  (zkCallback-4321-thread-5-processing-n:127.0.0.1:39694_solr) [n:127.0.0.1:39694_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@59b4358 name: ZooKeeperConnection Watcher:127.0.0.1:54040/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 3427953 WARN  (zkCallback-4321-thread-5-processing-n:127.0.0.1:39694_solr) [n:127.0.0.1:39694_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 3427985 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNullProxyUser
   [junit4]   2> 3427985 INFO  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.SolrDispatchFilter Error authenticating
   [junit4]   2> java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:213)
   [junit4]   2> 	at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> 	at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
   [junit4]   2> 	at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:134)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
   [junit4]   2> 	at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:243)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 3427986 WARN  (qtp444197900-30277) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.e.j.s.ServletHandler /solr/admin/collections
   [junit4]   2> org.apache.solr.common.SolrException: Error during request authentication, 
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:457)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:213)
   [junit4]   2> 	at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> 	at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
   [junit4]   2> 	at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:134)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
   [junit4]   2> 	at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:243)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
   [junit4]   2> 	... 29 more
   [junit4]   2> 3427988 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNullProxyUser
   [junit4]   2> 3428206 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidRemote
   [junit4]   2> 3428214 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[8EBC4DF65222051A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidRemote
   [junit4]   2> 3428595 INFO  (jetty-closer-4310-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@ad947b4{SSL,[ssl, http/1.1]}{127.0.0.1:0}
   [junit4]   2> 3428598 INFO  (jetty-closer-4310-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=2070017018
   [junit4]   2> 3428598 INFO  (jetty-closer-4310-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@79c82588{SSL,[ssl, http/1.1]}{127.0.0.1:0}
   [junit4]   2> 3428598 INFO  (jetty-closer-4310-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 3428599 INFO  (jetty-closer-4310-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@58a01694: rootName = solr_39694, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@252093a5
   [junit4]   2> 3428602 INFO  (jetty-closer-4310-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=88969757
   [junit4]   2> 3428602 INFO  (jetty-closer-4310-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 3428603 INFO  (jetty-closer-4310-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4b4c4b6f: rootName = solr_61472, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@1b5efcb0
   [junit4]   2> 3428612 INFO  (jetty-closer-4310-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 3428612 INFO  (jetty-closer-4310-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@470f074b: rootName = solr_39694, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@4af369af
   [junit4]   2> 3428614 INFO  (jetty-closer-4310-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 3428614 INFO  (jetty-closer-4310-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@15066683: rootName = solr_61472, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@4af369af
   [junit4]   2> 3428619 INFO  (jetty-closer-4310-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 3428619 INFO  (jetty-closer-4310-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@223d0c3c: rootName = solr_39694, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@50085238
   [junit4]   2> 3428619 INFO  (jetty-closer-4310-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 3428619 INFO  (jetty-closer-4310-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@26fe9308: rootName = solr_61472, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@50085238
   [junit4]   2> 3429483 INFO  (jetty-closer-4310-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 3429484 INFO  (coreCloseExecutor-11660-thread-1) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@22b76e9f
   [junit4]   2> 3429484 INFO  (coreCloseExecutor-11660-thread-1) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.forwardingCollection.shard1.replica_n1, tag=582446751
   [junit4]   2> 3429484 INFO  (coreCloseExecutor-11660-thread-1) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3e389e0e: rootName = solr_61472, domain = solr.core.forwardingCollection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.forwardingCollection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@e521c63
   [junit4]   2> 3429491 INFO  (coreCloseExecutor-11660-thread-1) [n:127.0.0.1:61472_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.forwardingCollection.shard1.leader, tag=582446751
   [junit4]   2> 3429494 ERROR (Thread[Thread-8355,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:61472_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 3429504 INFO  (jetty-closer-4310-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@22999a72{/solr,null,UNAVAILABLE}
   [junit4]   2> 3429856 WARN  (jetty-launcher-4309-thread-1-SendThread(127.0.0.1:54040)) [n:127.0.0.1:39694_solr    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15edd50ad1f0008 has expired
   [junit4]   2> 3429874 INFO  (zkCallback-4321-thread-6-processing-n:127.0.0.1:39694_solr) [n:127.0.0.1:39694_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 3429875 INFO  (jetty-closer-4310-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 3429876 INFO  (jetty-closer-4310-thread-2) [    ] o.a.s.c.Overseer Overseer (id=98759580420341766-127.0.0.1:39694_solr-n_0000000000) closing
   [junit4]   2> 3429877 INFO  (OverseerStateUpdate-98759580420341766-127.0.0.1:39694_solr-n_0000000000) [n:127.0.0.1:39694_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:39694_solr
   [junit4]   2> 3429880 WARN  (zkCallback-4321-thread-6-processing-n:127.0.0.1:39694_solr) [n:127.0.0.1:39694_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 3429882 ERROR (Thread[Thread-8357,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:39694_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 3429892 INFO  (jetty-closer-4310-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@285c447d{/solr,null,UNAVAILABLE}
   [junit4]   2> 3429895 ERROR (SUITE-TestSolrCloudWithSecureImpersonation-seed#[8EBC4DF65222051A]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 3429896 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[8EBC4DF65222051A]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:54040 54040
   [junit4]   2> 3439937 INFO  (Thread-8338) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:54040 54040
   [junit4]   2> 3439941 WARN  (Thread-8338) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/collections/forwardingCollection/state.json
   [junit4]   2> 	4	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	3	/solr/collections
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 
   [junit4]   2> 3439970 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[8EBC4DF65222051A]-worker) [    ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> Oct 02, 2017 1:40:46 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> 3458035 ERROR (jetty-launcher-4309-thread-2-EventThread) [n:127.0.0.1:61472_solr    ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
   [junit4]   2> java.lang.IllegalStateException: Client is not started
   [junit4]   2> 	at com.google.common.base.Preconditions.checkState(Preconditions.java:149)
   [junit4]   2> 	at org.apache.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:113)
   [junit4]   2> 	at org.apache.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:477)
   [junit4]   2> 	at org.apache.curator.framework.imps.NamespaceFacade.getZooKeeper(NamespaceFacade.java:112)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl$4.call(GetDataBuilderImpl.java:302)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl$4.call(GetDataBuilderImpl.java:291)
   [junit4]   2> 	at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:107)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2> 	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> Oct 02, 2017 1:41:06 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   [junit4]   2>    1) Thread[id=30319, name=jetty-launcher-4309-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2>         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2>         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2>         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2>         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> Oct 02, 2017 1:41:06 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=30319, name=jetty-launcher-4309-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2> 3460142 ERROR (jetty-launcher-4309-thread-1-EventThread) [n:127.0.0.1:39694_solr    ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2> 	at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2> 	at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2> 	at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2> 	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> 3460143 WARN  (jetty-launcher-4309-thread-1-EventThread) [n:127.0.0.1:39694_solr    ] o.a.c.ConnectionState Session expired event received
   [junit4]   2> 3460145 WARN  (jetty-launcher-4309-thread-1-SendThread(127.0.0.1:54040)) [n:127.0.0.1:39694_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
   [junit4]   2> Oct 02, 2017 1:41:06 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=749, maxMBSortInHeap=6.7051487759892225, sim=RandomSimilarity(queryNorm=false): {}, locale=en-IN, timezone=America/Anguilla
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=3,free=179558920,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [TestPHPSerializedResponseWriter, AssignTest, TestFieldCacheSortRandom, EnumFieldTest, TestExactSharedStatsCache, TestCustomDocTransformer, ConjunctionSolrSpellCheckerTest, OverseerRolesTest, ConcurrentDeleteAndCreateCollectionTest, ScriptEngineTest, TestDefaultStatsCache, V2ApiIntegrationTest, ShardSplitTest, QueryParsingTest, SolrIndexMetricsTest, TestDFRSimilarityFactory, SolrCoreTest, TolerantUpdateProcessorTest, HdfsRecoverLeaseTest, UtilsToolTest, TestSolrCloudWithHadoopAuthPlugin, TestUnifiedSolrHighlighter, TestScoreJoinQPNoScore, TestCollectionAPI, TestInitParams, TestRetrieveFieldsOptimizer, TestDocTermOrds, TestReRankQParserPlugin, TestFileDictionaryLookup, TestFuzzyAnalyzedSuggestions, CursorMarkTest, TestDFISimilarityFactory, TestStressRecovery, TestRawResponseWriter, TestRandomCollapseQParserPlugin, TestManagedSchemaThreadSafety, DirectUpdateHandlerTest, TestIndexingPerformance, TestChildDocTransformer, DistributedExpandComponentTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, BinaryUpdateRequestHandlerTest, ReplicaListTransformerTest, TestReplicationHandler, DeleteStatusTest, TestMiniSolrCloudClusterSSL, SimpleMLTQParserTest, TestDownShardTolerantSearch, TestXIncludeConfig, AnalysisErrorHandlingTest, SortByFunctionTest, DefaultValueUpdateProcessorTest, CSVRequestHandlerTest, HdfsRestartWhileUpdatingTest, SolrCLIZkUtilsTest, TestWriterPerf, TestLFUCache, HighlighterConfigTest, StatsReloadRaceTest, ZkCLITest, TestJsonFacetRefinement, TestAuthorizationFramework, IgnoreCommitOptimizeUpdateProcessorFactoryTest, SoftAutoCommitTest, TestGraphTermsQParserPlugin, BigEndianAscendingWordDeserializerTest, TlogReplayBufferedWhileIndexingTest, LeaderElectionTest, XsltUpdateRequestHandlerTest, TestSurroundQueryParser, TestLMDirichletSimilarityFactory, TestConfigSetsAPIZkFailure, SuggesterTest, TestSolrCoreProperties, TestUninvertingReader, RulesTest, TestCloudManagedSchema, SuggesterWFSTTest, TestPullReplicaErrorHandling, UniqFieldsUpdateProcessorFactoryTest, PreAnalyzedUpdateProcessorTest, CheckHdfsIndexTest, TestLazyCores, TestLocalFSCloudBackupRestore, SecurityConfHandlerTest, TemplateUpdateProcessorTest, TestSweetSpotSimilarityFactory, SearchHandlerTest, MultiThreadedOCPTest, TestCursorMarkWithoutUniqueKey, TestSolrCLIRunExample, AnalyticsMergeStrategyTest, TestFaceting, TestSubQueryTransformerCrossCore, RequiredFieldsTest, CollectionStateFormat2Test, TestPhraseSuggestions, TestHighFrequencyDictionaryFactory, TestReloadDeadlock, TestOmitPositions, BitVectorTest, TestPerFieldSimilarityWithDefaultOverride, TestCustomStream, TestSchemaResource, PathHierarchyTokenizerFactoryTest, ReplaceNodeTest, TestOverriddenPrefixQueryForCustomFieldType, CloneFieldUpdateProcessorFactoryTest, SimpleFacetsTest, CircularListTest, RAMDirectoryFactoryTest, IndexBasedSpellCheckerTest, MergeStrategyTest, DocValuesNotIndexedTest, TestRuleBasedAuthorizationPlugin, HdfsDirectoryFactoryTest, TestUtils, TestSQLHandlerNonCloud, TestFieldTypeResource, OverseerTest, TestReplicaProperties, TestFieldCacheReopen, TestJsonRequest, TestConfigSets, TestRestoreCore, AlternateDirectoryTest, TestTolerantSearch, TestExpandComponent, MinimalSchemaTest, SolrInfoBeanTest, AutoScalingHandlerTest, TestUpdate, TestStressUserVersions, HighlighterTest, TestCollectionAPIs, CoreAdminHandlerTest, SSLMigrationTest, MultiTermTest, DistanceUnitsTest, TestReloadAndDeleteDocs, TestNumericTerms32, TestHdfsCloudBackupRestore, TestQuerySenderNoQuery, TestFunctionQuery, BlockJoinFacetSimpleTest, TestPivotHelperCode, HdfsChaosMonkeyNothingIsSafeTest, TestLeaderElectionZkExpiry, FullHLLTest, NotRequiredUniqueKeyTest, DistributedFacetPivotSmallAdvancedTest, MissingSegmentRecoveryTest, TestSolrConfigHandlerConcurrent, UpdateLogTest, ClassificationUpdateProcessorIntegrationTest, ResponseLogComponentTest, DataDrivenBlockJoinTest, TestFoldingMultitermQuery, TimeZoneUtilsTest, SpatialHeatmapFacetsTest, SharedFSAutoReplicaFailoverUtilsTest, TestHashPartitioner, TestMinMaxOnMultiValuedField, TestSort, StressHdfsTest, OutputWriterTest, HdfsUnloadDistributedZkTest, TestGeoJSONResponseWriter, DeleteReplicaTest, ConfigureRecoveryStrategyTest, TestCollapseQParserPlugin, TestSimpleTextCodec, CursorPagingTest, TestMaxScoreQueryParser, SolrMetricReporterTest, TestXmlQParser, WordBreakSolrSpellCheckerTest, TestReplicationHandlerBackup, TestConfigReload, BasicDistributedZk2Test, UnloadDistributedZkTest, SyncSliceTest, LeaderElectionIntegrationTest, RecoveryZkTest, ClusterStateUpdateTest, TestRandomFaceting, ZkSolrClientTest, AutoCommitTest, TestDistributedGrouping, TestRecovery, DistributedSpellCheckComponentTest, TestRealTimeGet, TestJoin, HardAutoCommitTest, TestCoreContainer, TestSolr4Spatial, TestGroupingSearch, SolrCmdDistributorTest, QueryElevationComponentTest, PeerSyncTest, SignatureUpdateProcessorFactoryTest, DocValuesTest, XmlUpdateRequestHandlerTest, RequestHandlersTest, TermVectorComponentTest, FastVectorHighlighterTest, RegexBoostProcessorTest, IndexSchemaTest, TestCollationField, ReturnFieldsTest, TestAnalyzedSuggestions, SOLR749Test, TestQuerySenderListener, CopyFieldTest, SolrIndexConfigTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, TestDocSet, TestBinaryField, NumericFieldsTest, TestElisionMultitermQuery, TestConfig, ExternalFileFieldSortTest, DirectSolrConnectionTest, TestCodecSupport, SynonymTokenizerTest, TestIBSimilarityFactory, ResourceLoaderTest, OpenExchangeRatesOrgProviderTest, DateMathParserTest, PrimUtilsTest, DateFieldTest, DOMUtilTest, ClusterStateTest, TestDocumentBuilder, SliceStateTest, UUIDFieldTest, DistributedIntervalFacetingTest, SolrTestCaseJ4Test, TestHighlightDedupGrouping, TestSimpleTrackingShardHandler, TestLuceneIndexBackCompat, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerConstructors, TestEmbeddedSolrServerSchemaAPI, TestJettySolrRunner, ConnectionReuseTest, ActionThrottleTest, AliasIntegrationTest, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrBootstrapTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, CollectionReloadTest, CollectionsAPIAsyncDistributedZkTest, CustomCollectionTest, DistribDocExpirationUpdateProcessorTest, DistributedQueueTest, HealthCheckHandlerTest, LeaderElectionContextKeyTest, LeaderFailoverAfterPartitionTest, LeaderInitiatedRecoveryOnShardRestartTest, MigrateRouteKeyTest, NodeMutatorTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerModifyCollectionTest, OverseerTaskQueueTest, PeerSyncReplicationTest, RemoteQueryErrorTest, RestartWhileUpdatingTest, RollingRestartTest, SolrCloudExampleTest, TestCloudDeleteByQuery, TestCloudInspectUtil, TestCloudJSONFacetJoinDomain, TestCloudPseudoReturnFields, TestCloudRecovery, TestClusterProperties, TestCollectionsAPIViaSolrCloudCluster, TestConfigSetsAPIExclusivity, TestDeleteCollectionOnDownNodes, TestDistribDocBasedVersion, TestLeaderElectionWithEmptyReplica, TestPrepRecovery, TestShortCircuitedRequests, TestSolrCloudWithKerberosAlt, TestSolrCloudWithSecureImpersonation]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=8EBC4DF65222051A -Dtests.slow=true -Dtests.locale=en-IN -Dtests.timezone=America/Anguilla -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J0 | TestSolrCloudWithSecureImpersonation (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   [junit4]    >    1) Thread[id=30319, name=jetty-launcher-4309-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8EBC4DF65222051A]:0)
   [junit4] Completed [567/734 (1!)] on J0 in 60.95s, 12 tests, 1 error <<< FAILURES!

[...truncated 37784 lines...]
-documentation-lint:
     [echo] checking for broken html...
    [jtidy] Checking for broken html (such as invalid tags)...
   [delete] Deleting directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/build/jtidy_tmp
     [echo] Checking for broken links...
     [exec] 
     [exec] Crawl/parse...
     [exec] 
     [exec] Verify...
     [exec] 
     [exec] file:///build/docs/sandbox/org/apache/lucene/document/FloatPointNearestNeighbor.html
     [exec]   BROKEN LINK: file:///build/docs/core/org/apache/lucene/document/FloatPointNearestNeighbor.NearestHit.html
     [exec]   BROKEN LINK: file:///build/docs/core/org/apache/lucene/document/FloatPointNearestNeighbor.NearestHit.html
     [exec] 
     [exec] Broken javadocs links were found! Common root causes:
     [exec] * A typo of some sort for manually created links.
     [exec] * Public methods referencing non-public classes in their signature.

BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:826: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:101: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/build.xml:142: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/build.xml:155: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/common-build.xml:2570: exec returned: 1

Total time: 100 minutes 40 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1448 - Failure!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1448/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

2 tests failed.
FAILED:  org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingDocAbsent

Error Message:
Did not find the expected number of groups for field intGSF expected:<4> but was:<3>

Stack Trace:
java.lang.AssertionError: Did not find the expected number of groups for field intGSF expected:<4> but was:<3>
	at __randomizedtesting.SeedInfo.seed([B44C3FCFFA90B617:6AFBE4471FC1F6D2]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingDocAbsent(DocValuesNotIndexedTest.java:305)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:     1) Thread[id=23031, name=jetty-launcher-3705-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)    2) Thread[id=23041, name=jetty-launcher-3705-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   1) Thread[id=23031, name=jetty-launcher-3705-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   2) Thread[id=23041, name=jetty-launcher-3705-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
	at __randomizedtesting.SeedInfo.seed([B44C3FCFFA90B617]:0)




Build Log:
[...truncated 13089 lines...]
   [junit4] Suite: org.apache.solr.cloud.DocValuesNotIndexedTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B44C3FCFFA90B617-001/init-core-data-001
   [junit4]   2> 2393580 WARN  (SUITE-DocValuesNotIndexedTest-seed#[B44C3FCFFA90B617]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=7 numCloses=7
   [junit4]   2> 2393581 INFO  (SUITE-DocValuesNotIndexedTest-seed#[B44C3FCFFA90B617]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 2393582 INFO  (SUITE-DocValuesNotIndexedTest-seed#[B44C3FCFFA90B617]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2393582 INFO  (SUITE-DocValuesNotIndexedTest-seed#[B44C3FCFFA90B617]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2393583 INFO  (SUITE-DocValuesNotIndexedTest-seed#[B44C3FCFFA90B617]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B44C3FCFFA90B617-001/tempDir-001
   [junit4]   2> 2393583 INFO  (SUITE-DocValuesNotIndexedTest-seed#[B44C3FCFFA90B617]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2393583 INFO  (Thread-4241) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2393584 INFO  (Thread-4241) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2393586 ERROR (Thread-4241) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2393683 INFO  (SUITE-DocValuesNotIndexedTest-seed#[B44C3FCFFA90B617]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:60627
   [junit4]   2> 2393696 INFO  (jetty-launcher-2971-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2393696 INFO  (jetty-launcher-2971-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2393700 INFO  (jetty-launcher-2971-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1410eeb6{/solr,null,AVAILABLE}
   [junit4]   2> 2393701 INFO  (jetty-launcher-2971-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@262d2e6e{/solr,null,AVAILABLE}
   [junit4]   2> 2393701 INFO  (jetty-launcher-2971-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@53de0f2c{HTTP/1.1,[http/1.1]}{127.0.0.1:48156}
   [junit4]   2> 2393701 INFO  (jetty-launcher-2971-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@27f63594{HTTP/1.1,[http/1.1]}{127.0.0.1:62879}
   [junit4]   2> 2393701 INFO  (jetty-launcher-2971-thread-1) [    ] o.e.j.s.Server Started @2396873ms
   [junit4]   2> 2393701 INFO  (jetty-launcher-2971-thread-2) [    ] o.e.j.s.Server Started @2396873ms
   [junit4]   2> 2393701 INFO  (jetty-launcher-2971-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=62879}
   [junit4]   2> 2393701 INFO  (jetty-launcher-2971-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=48156}
   [junit4]   2> 2393702 ERROR (jetty-launcher-2971-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2393702 ERROR (jetty-launcher-2971-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2393702 INFO  (jetty-launcher-2971-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 2393703 INFO  (jetty-launcher-2971-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2393703 INFO  (jetty-launcher-2971-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 2393704 INFO  (jetty-launcher-2971-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2393704 INFO  (jetty-launcher-2971-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2393704 INFO  (jetty-launcher-2971-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-02T02:41:11.365Z
   [junit4]   2> 2393704 INFO  (jetty-launcher-2971-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2393704 INFO  (jetty-launcher-2971-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-02T02:41:11.365Z
   [junit4]   2> 2393710 INFO  (jetty-launcher-2971-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2393710 INFO  (jetty-launcher-2971-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2393740 INFO  (jetty-launcher-2971-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60627/solr
   [junit4]   2> 2393742 INFO  (jetty-launcher-2971-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60627/solr
   [junit4]   2> 2393884 INFO  (jetty-launcher-2971-thread-1) [n:127.0.0.1:62879_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2393885 INFO  (jetty-launcher-2971-thread-1) [n:127.0.0.1:62879_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:62879_solr
   [junit4]   2> 2393886 INFO  (jetty-launcher-2971-thread-1) [n:127.0.0.1:62879_solr    ] o.a.s.c.Overseer Overseer (id=98756989501177862-127.0.0.1:62879_solr-n_0000000000) starting
   [junit4]   2> 2393890 INFO  (jetty-launcher-2971-thread-2) [n:127.0.0.1:48156_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2393892 INFO  (jetty-launcher-2971-thread-2) [n:127.0.0.1:48156_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48156_solr
   [junit4]   2> 2393894 INFO  (zkCallback-2982-thread-1-processing-n:127.0.0.1:48156_solr) [n:127.0.0.1:48156_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2393894 INFO  (zkCallback-2983-thread-1-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2393899 INFO  (jetty-launcher-2971-thread-1) [n:127.0.0.1:62879_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62879_solr
   [junit4]   2> 2393900 INFO  (zkCallback-2982-thread-1-processing-n:127.0.0.1:48156_solr) [n:127.0.0.1:48156_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2393900 INFO  (zkCallback-2983-thread-1-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2394005 INFO  (jetty-launcher-2971-thread-2) [n:127.0.0.1:48156_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48156.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@66f46c05
   [junit4]   2> 2394010 INFO  (jetty-launcher-2971-thread-2) [n:127.0.0.1:48156_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48156.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@66f46c05
   [junit4]   2> 2394010 INFO  (jetty-launcher-2971-thread-2) [n:127.0.0.1:48156_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48156.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@66f46c05
   [junit4]   2> 2394010 INFO  (jetty-launcher-2971-thread-2) [n:127.0.0.1:48156_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B44C3FCFFA90B617-001/tempDir-001/node2/.
   [junit4]   2> 2394030 INFO  (jetty-launcher-2971-thread-1) [n:127.0.0.1:62879_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62879.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@66f46c05
   [junit4]   2> 2394038 INFO  (jetty-launcher-2971-thread-1) [n:127.0.0.1:62879_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62879.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@66f46c05
   [junit4]   2> 2394039 INFO  (jetty-launcher-2971-thread-1) [n:127.0.0.1:62879_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62879.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@66f46c05
   [junit4]   2> 2394040 INFO  (jetty-launcher-2971-thread-1) [n:127.0.0.1:62879_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B44C3FCFFA90B617-001/tempDir-001/node1/.
   [junit4]   2> 2394072 INFO  (SUITE-DocValuesNotIndexedTest-seed#[B44C3FCFFA90B617]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2394074 INFO  (SUITE-DocValuesNotIndexedTest-seed#[B44C3FCFFA90B617]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60627/solr ready
   [junit4]   2> 2394082 INFO  (qtp195727981-18616) [n:127.0.0.1:62879_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&maxShardsPerNode=2&name=dv_coll&nrtReplicas=1&action=CREATE&numShards=4&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2394086 INFO  (OverseerThreadFactory-8632-thread-1-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.c.CreateCollectionCmd Create collection dv_coll
   [junit4]   2> 2394202 INFO  (OverseerStateUpdate-98756989501177862-127.0.0.1:62879_solr-n_0000000000) [n:127.0.0.1:62879_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"dv_coll",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"dv_coll_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:62879/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 2394207 INFO  (OverseerStateUpdate-98756989501177862-127.0.0.1:62879_solr-n_0000000000) [n:127.0.0.1:62879_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"dv_coll",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"dv_coll_shard2_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:48156/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 2394212 INFO  (OverseerStateUpdate-98756989501177862-127.0.0.1:62879_solr-n_0000000000) [n:127.0.0.1:62879_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"dv_coll",
   [junit4]   2>   "shard":"shard3",
   [junit4]   2>   "core":"dv_coll_shard3_replica_n4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:62879/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 2394219 INFO  (OverseerStateUpdate-98756989501177862-127.0.0.1:62879_solr-n_0000000000) [n:127.0.0.1:62879_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"dv_coll",
   [junit4]   2>   "shard":"shard4",
   [junit4]   2>   "core":"dv_coll_shard4_replica_n6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:48156/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 2394418 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf1&newCollection=true&name=dv_coll_shard1_replica_n1&action=CREATE&numShards=4&collection=dv_coll&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2394419 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf1&newCollection=true&name=dv_coll_shard3_replica_n4&action=CREATE&numShards=4&collection=dv_coll&shard=shard3&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2394419 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2394421 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf1&newCollection=true&name=dv_coll_shard4_replica_n6&action=CREATE&numShards=4&collection=dv_coll&shard=shard4&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2394421 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf1&newCollection=true&name=dv_coll_shard2_replica_n2&action=CREATE&numShards=4&collection=dv_coll&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2394421 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2394531 INFO  (zkCallback-2983-thread-1-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2394531 INFO  (zkCallback-2983-thread-2-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2394531 INFO  (zkCallback-2982-thread-1-processing-n:127.0.0.1:48156_solr) [n:127.0.0.1:48156_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2394532 INFO  (zkCallback-2982-thread-2-processing-n:127.0.0.1:48156_solr) [n:127.0.0.1:48156_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2395444 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2395493 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.s.IndexSchema [dv_coll_shard2_replica_n2] Schema name=minimal
   [junit4]   2> 2395494 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2395496 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2395507 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2395507 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard2_replica_n2' using configuration from collection dv_coll, trusted=true
   [junit4]   2> 2395508 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48156.solr.core.dv_coll.shard2.replica_n2' (registry 'solr.core.dv_coll.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@66f46c05
   [junit4]   2> 2395508 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2395508 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrCore [[dv_coll_shard2_replica_n2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B44C3FCFFA90B617-001/tempDir-001/node2/dv_coll_shard2_replica_n2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B44C3FCFFA90B617-001/tempDir-001/node2/./dv_coll_shard2_replica_n2/data/]
   [junit4]   2> 2395521 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.IndexSchema [dv_coll_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 2395521 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.IndexSchema [dv_coll_shard3_replica_n4] Schema name=minimal
   [junit4]   2> 2395524 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2395524 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard1_replica_n1' using configuration from collection dv_coll, trusted=true
   [junit4]   2> 2395524 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2395524 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard3_replica_n4' using configuration from collection dv_coll, trusted=true
   [junit4]   2> 2395525 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62879.solr.core.dv_coll.shard1.replica_n1' (registry 'solr.core.dv_coll.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@66f46c05
   [junit4]   2> 2395525 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62879.solr.core.dv_coll.shard3.replica_n4' (registry 'solr.core.dv_coll.shard3.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@66f46c05
   [junit4]   2> 2395525 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2395525 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2395525 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [[dv_coll_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B44C3FCFFA90B617-001/tempDir-001/node1/dv_coll_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B44C3FCFFA90B617-001/tempDir-001/node1/./dv_coll_shard1_replica_n1/data/]
   [junit4]   2> 2395525 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SolrCore [[dv_coll_shard3_replica_n4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B44C3FCFFA90B617-001/tempDir-001/node1/dv_coll_shard3_replica_n4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B44C3FCFFA90B617-001/tempDir-001/node1/./dv_coll_shard3_replica_n4/data/]
   [junit4]   2> 2395576 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2395609 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2395609 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2395610 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2395611 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2395612 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@4910859a[dv_coll_shard2_replica_n2] main]
   [junit4]   2> 2395617 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.s.IndexSchema [dv_coll_shard4_replica_n6] Schema name=minimal
   [junit4]   2> 2395619 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2395619 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard4_replica_n6' using configuration from collection dv_coll, trusted=true
   [junit4]   2> 2395620 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48156.solr.core.dv_coll.shard4.replica_n6' (registry 'solr.core.dv_coll.shard4.replica_n6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@66f46c05
   [junit4]   2> 2395620 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2395620 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.SolrCore [[dv_coll_shard4_replica_n6] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B44C3FCFFA90B617-001/tempDir-001/node2/dv_coll_shard4_replica_n6], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B44C3FCFFA90B617-001/tempDir-001/node2/./dv_coll_shard4_replica_n6/data/]
   [junit4]   2> 2395629 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2395630 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2395630 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 2395632 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 2395632 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2395632 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580111834165280768
   [junit4]   2> 2395634 INFO  (searcherExecutor-8637-thread-1-processing-n:127.0.0.1:48156_solr x:dv_coll_shard2_replica_n2 s:shard2 c:dv_coll r:core_node5) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrCore [dv_coll_shard2_replica_n2] Registered new searcher Searcher@4910859a[dv_coll_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2395656 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2395656 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2395657 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2395658 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2395659 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@76f595b[dv_coll_shard1_replica_n1] main]
   [junit4]   2> 2395661 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2395661 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2395662 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2395662 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2395664 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@69246848[dv_coll_shard3_replica_n4] main]
   [junit4]   2> 2395666 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2395673 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2395674 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2395674 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 2395678 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2395678 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 2395679 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2395679 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2395679 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:48156/solr/dv_coll_shard2_replica_n2/
   [junit4]   2> 2395679 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2395679 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.SyncStrategy http://127.0.0.1:48156/solr/dv_coll_shard2_replica_n2/ has no replicas
   [junit4]   2> 2395679 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2395681 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 2395684 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:48156/solr/dv_coll_shard2_replica_n2/ shard2
   [junit4]   2> 2395684 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2395684 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580111834219806720
   [junit4]   2> 2395689 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 2395691 INFO  (searcherExecutor-8638-thread-1-processing-n:127.0.0.1:62879_solr x:dv_coll_shard1_replica_n1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [dv_coll_shard1_replica_n1] Registered new searcher Searcher@76f595b[dv_coll_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2395691 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2395691 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580111834227146752
   [junit4]   2> 2395695 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2395695 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2395695 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:62879/solr/dv_coll_shard1_replica_n1/
   [junit4]   2> 2395696 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2395696 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:62879/solr/dv_coll_shard1_replica_n1/ has no replicas
   [junit4]   2> 2395696 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2395697 INFO  (searcherExecutor-8639-thread-1-processing-n:127.0.0.1:62879_solr x:dv_coll_shard3_replica_n4 s:shard3 c:dv_coll r:core_node7) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SolrCore [dv_coll_shard3_replica_n4] Registered new searcher Searcher@69246848[dv_coll_shard3_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2395708 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2395708 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2395708 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:62879/solr/dv_coll_shard3_replica_n4/
   [junit4]   2> 2395708 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2395711 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SyncStrategy http://127.0.0.1:62879/solr/dv_coll_shard3_replica_n4/ has no replicas
   [junit4]   2> 2395711 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2395715 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:62879/solr/dv_coll_shard1_replica_n1/ shard1
   [junit4]   2> 2395715 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2395716 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2395721 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2395721 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2395722 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.s.SolrIndexSearcher Opening [Searcher@beee979[dv_coll_shard4_replica_n6] main]
   [junit4]   2> 2395723 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:62879/solr/dv_coll_shard3_replica_n4/ shard3
   [junit4]   2> 2395724 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2395724 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2395727 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 2395728 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 2395728 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2395729 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580111834266992640
   [junit4]   2> 2395729 INFO  (searcherExecutor-8644-thread-1-processing-n:127.0.0.1:48156_solr x:dv_coll_shard4_replica_n6 s:shard4 c:dv_coll r:core_node8) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.SolrCore [dv_coll_shard4_replica_n6] Registered new searcher Searcher@beee979[dv_coll_shard4_replica_n6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2395747 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2395747 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2395747 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:48156/solr/dv_coll_shard4_replica_n6/
   [junit4]   2> 2395747 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2395747 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.SyncStrategy http://127.0.0.1:48156/solr/dv_coll_shard4_replica_n6/ has no replicas
   [junit4]   2> 2395747 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2395757 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:48156/solr/dv_coll_shard4_replica_n6/ shard4
   [junit4]   2> 2395874 INFO  (zkCallback-2982-thread-1-processing-n:127.0.0.1:48156_solr) [n:127.0.0.1:48156_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2395875 INFO  (zkCallback-2983-thread-2-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2395875 INFO  (zkCallback-2983-thread-1-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2395875 INFO  (zkCallback-2982-thread-2-processing-n:127.0.0.1:48156_solr) [n:127.0.0.1:48156_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2395878 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2395884 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2395884 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf1&newCollection=true&name=dv_coll_shard3_replica_n4&action=CREATE&numShards=4&collection=dv_coll&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1465
   [junit4]   2> 2395892 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2395898 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf1&newCollection=true&name=dv_coll_shard1_replica_n1&action=CREATE&numShards=4&collection=dv_coll&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1480
   [junit4]   2> 2395900 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf1&newCollection=true&name=dv_coll_shard2_replica_n2&action=CREATE&numShards=4&collection=dv_coll&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1479
   [junit4]   2> 2395913 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2395918 INFO  (qtp2126010706-18611) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf1&newCollection=true&name=dv_coll_shard4_replica_n6&action=CREATE&numShards=4&collection=dv_coll&shard=shard4&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1497
   [junit4]   2> 2395951 INFO  (qtp195727981-18616) [n:127.0.0.1:62879_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 2396069 INFO  (zkCallback-2982-thread-2-processing-n:127.0.0.1:48156_solr) [n:127.0.0.1:48156_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2396070 INFO  (zkCallback-2982-thread-1-processing-n:127.0.0.1:48156_solr) [n:127.0.0.1:48156_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2396078 INFO  (zkCallback-2983-thread-1-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2396078 INFO  (zkCallback-2983-thread-2-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2396090 INFO  (OverseerCollectionConfigSetProcessor-98756989501177862-127.0.0.1:62879_solr-n_0000000000) [n:127.0.0.1:62879_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2396954 INFO  (qtp195727981-18616) [n:127.0.0.1:62879_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&maxShardsPerNode=2&name=dv_coll&nrtReplicas=1&action=CREATE&numShards=4&wt=javabin&version=2} status=0 QTime=2872
   [junit4]   2> 2396962 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.SchemaManager managed schema loaded . version : 0 
   [junit4]   2> 2396965 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.IndexSchema [dv_coll_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 2397004 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2397021 INFO  (zkCallback-2983-thread-1-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
   [junit4]   2> 2397021 INFO  (zkCallback-2983-thread-2-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
   [junit4]   2> 2397021 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.ZkController Persisted config data to node /configs/conf1/managed-schema 
   [junit4]   2> 2397026 INFO  (zkCallback-2982-thread-1-processing-n:127.0.0.1:48156_solr) [n:127.0.0.1:48156_solr    ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
   [junit4]   2> 2397027 INFO  (zkCallback-2982-thread-1-processing-n:127.0.0.1:48156_solr) [n:127.0.0.1:48156_solr    ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
   [junit4]   2> 2397029 INFO  (zkCallback-2982-thread-1-processing-n:127.0.0.1:48156_solr) [n:127.0.0.1:48156_solr    ] o.a.s.s.IndexSchema [dv_coll_shard4_replica_n6] Schema name=minimal
   [junit4]   2> 2397029 INFO  (zkCallback-2982-thread-2-processing-n:127.0.0.1:48156_solr) [n:127.0.0.1:48156_solr    ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
   [junit4]   2> 2397030 INFO  (zkCallback-2982-thread-2-processing-n:127.0.0.1:48156_solr) [n:127.0.0.1:48156_solr    ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
   [junit4]   2> 2397033 INFO  (zkCallback-2982-thread-2-processing-n:127.0.0.1:48156_solr) [n:127.0.0.1:48156_solr    ] o.a.s.s.IndexSchema [dv_coll_shard2_replica_n2] Schema name=minimal
   [junit4]   2> 2397034 INFO  (zkCallback-2982-thread-1-processing-n:127.0.0.1:48156_solr) [n:127.0.0.1:48156_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2397034 INFO  (zkCallback-2982-thread-1-processing-n:127.0.0.1:48156_solr) [n:127.0.0.1:48156_solr    ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 7 ms
   [junit4]   2> 2397034 INFO  (zkCallback-2983-thread-1-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
   [junit4]   2> 2397037 INFO  (zkCallback-2983-thread-1-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.s.IndexSchema [dv_coll_shard3_replica_n4] Schema name=minimal
   [junit4]   2> 2397043 INFO  (zkCallback-2982-thread-2-processing-n:127.0.0.1:48156_solr) [n:127.0.0.1:48156_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2397043 INFO  (zkCallback-2982-thread-2-processing-n:127.0.0.1:48156_solr) [n:127.0.0.1:48156_solr    ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 12 ms
   [junit4]   2> 2397045 INFO  (Thread-4260) [n:127.0.0.1:48156_solr    ] o.a.s.c.SolrCore config update listener called for core dv_coll_shard4_replica_n6
   [junit4]   2> 2397046 INFO  (Thread-4260) [n:127.0.0.1:48156_solr    ] o.a.s.c.SolrCore config update listener called for core dv_coll_shard2_replica_n2
   [junit4]   2> 2397047 INFO  (zkCallback-2983-thread-1-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2397047 INFO  (zkCallback-2983-thread-1-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 12 ms
   [junit4]   2> 2397048 INFO  (Thread-4261) [n:127.0.0.1:62879_solr    ] o.a.s.c.SolrCore config update listener called for core dv_coll_shard1_replica_n1
   [junit4]   2> 2397051 INFO  (Thread-4261) [n:127.0.0.1:62879_solr    ] o.a.s.c.SolrCore core reload dv_coll_shard1_replica_n1
   [junit4]   2> 2397060 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2397069 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.IndexSchema [dv_coll_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 2397074 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2397074 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.CoreContainer Reloading SolrCore 'dv_coll_shard1_replica_n1' using configuration from collection dv_coll
   [junit4]   2> 2397161 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62879.solr.core.dv_coll.shard1.replica_n1' (registry 'solr.core.dv_coll.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@66f46c05
   [junit4]   2> 2397161 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [[dv_coll_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B44C3FCFFA90B617-001/tempDir-001/node1/dv_coll_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B44C3FCFFA90B617-001/tempDir-001/node1/./dv_coll_shard1_replica_n1/data/]
   [junit4]   2> 2397241 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2397241 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2397243 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6376189c[dv_coll_shard1_replica_n1] main]
   [junit4]   2> 2397249 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2397249 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2397250 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 2397260 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 1 is already the latest
   [junit4]   2> 2397260 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2397260 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580111835872362496
   [junit4]   2> 2397263 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 2397267 INFO  (searcherExecutor-8657-thread-1-processing-n:127.0.0.1:62879_solr x:dv_coll_shard1_replica_n1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [dv_coll_shard1_replica_n1] Registered new searcher Searcher@6376189c[dv_coll_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2397267 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7c0c5716[dv_coll_shard1_replica_n1] main]
   [junit4]   2> 2397267 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ManagedIndexSchema Waiting up to 599 secs for 3 replicas to apply schema update version 1 for collection dv_coll
   [junit4]   2> 2397270 INFO  (zkCallback-2983-thread-2-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
   [junit4]   2> 2397274 INFO  (zkCallback-2983-thread-2-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.s.IndexSchema [dv_coll_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 2397274 INFO  (qtp2126010706-18610) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.S.Request [dv_coll_shard2_replica_n2]  webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=1&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2397279 INFO  (zkCallback-2983-thread-2-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2397279 INFO  (zkCallback-2983-thread-2-processing-n:127.0.0.1:62879_solr) [n:127.0.0.1:62879_solr    ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 8 ms
   [junit4]   2> 2397279 INFO  (searcherExecutor-8657-thread-1-processing-n:127.0.0.1:62879_solr x:dv_coll_shard1_replica_n1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [dv_coll_shard1_replica_n1] Registered new searcher Searcher@7c0c5716[dv_coll_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2397286 INFO  (qtp195727981-18617) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.S.Request [dv_coll_shard3_replica_n4]  webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=1&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2397288 INFO  (Thread-4261) [n:127.0.0.1:62879_solr    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2397292 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.S.Request [dv_coll_shard4_replica_n6]  webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=1&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2397296 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.ManagedIndexSchema Took 28.0ms for 3 replicas to apply schema update version 1 for collection dv_coll
   [junit4]   2> 2397296 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.S.Request [dv_coll_shard1_replica_n1]  webapp=/solr path=/schema params={wt=javabin&version=2} status=0 QTime=337
   [junit4]   2> 2397296 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [dv_coll_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@6f2f0d46
   [junit4]   2> 2397296 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.dv_coll.shard1.replica_n1, tag=1865354566
   [junit4]   2> 2397296 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3896bc0a: rootName = solr_62879, domain = solr.core.dv_coll.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.dv_coll.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@58961d90
   [junit4]   2> 2397299 INFO  (Thread-4261) [n:127.0.0.1:62879_solr    ] o.a.s.s.IndexSchema [dv_coll_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 2397303 INFO  (Thread-4261) [n:127.0.0.1:62879_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2397303 INFO  (Thread-4261) [n:127.0.0.1:62879_solr    ] o.a.s.c.CoreContainer Reloading SolrCore 'dv_coll_shard1_replica_n1' using configuration from collection dv_coll
   [junit4]   2> 2397304 INFO  (qtp195727981-18624) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.dv_coll.shard1.leader, tag=1865354566
   [junit4]   2> 2397429 INFO  (TEST-DocValuesNotIndexedTest.testDistribFaceting-seed#[B44C3FCFFA90B617]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDistribFaceting
   [junit4]   2> 2397437 INFO  (qtp2126010706-18615) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica_n6]  webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:62879/solr/dv_coll_shard3_replica_n4/&wt=javabin&version=2}{deleteByQuery=*:* (-1580111836055863296)} 0 2
   [junit4]   2> 2397450 INFO  (Thread-4261) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62879.solr.core.dv_coll.shard1.replica_n1' (registry 'solr.core.dv_coll.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@66f46c05
   [junit4]   2> 2397450 INFO  (Thread-4261) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [[dv_coll_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B44C3FCFFA90B617-001/tempDir-001/node1/dv_coll_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_B44C3FCFFA90B617-001/tempDir-001/node1/./dv_coll_shard1_replica_n1/data/]
   [junit4]   2> 2397501 INFO  (qtp195727981-18616) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica_n1]  webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:62879/solr/dv_coll_shard3_replica_n4/&wt=javabin&version=2}{deleteByQuery=*:* (-1580111836064251904)} 0 57
   [junit4]   2> 2397505 INFO  (qtp2126010706-18614) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica_n2]  webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:62879/solr/dv_coll_shard3_replica_n4/&wt=javabin&version=2}{deleteByQuery=*:* (-1580111836054814720)} 0 70
   [junit4]   2> 2397510 INFO  (qtp195727981-18619) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica_n4]  webapp=/solr path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1580111836050620416)} 0 80
   [junit4]   2> 2397515 INFO  (qtp2126010706-18608) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1580111836139749376,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2397515 INFO  (qtp2126010706-18608) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 2397515 INFO  (qtp2126010706-18608) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.s.SolrIndexSearcher Opening [Searcher@29f63872[dv_coll_shard4_replica_n6] main]
   [junit4]   2> 2397516 INFO  (qtp2126010706-18608) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2397516 INFO  (qtp195727981-18617) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1580111836140797952,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2397516 INFO  (qtp195727981-18617) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 2397516 INFO  (qtp195727981-18617) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@4ba24c32[dv_coll_shard3_replica_n4] main]
   [junit4]   2> 2397516 INFO  (qtp195727981-18617) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2397516 INFO  (searcherExecutor-8644-thread-1-processing-n:127.0.0.1:48156_solr x:dv_coll_shard4_replica_n6 s:shard4 c:dv_coll r:core_node8) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.c.SolrCore [dv_coll_shard4_replica_n6] Registered new searcher Searcher@29f63872[dv_coll_shard4_replica_n6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2397516 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1580111836140797952,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2397517 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 2397517 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2397517 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica_n1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:62879/solr/dv_coll_shard1_replica_n1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
   [junit4]   2> 2397517 INFO  (qtp2126010706-18608) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica_n6]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:62879/solr/dv_coll_shard1_replica_n1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 2
   [junit4]   2> 2397519 INFO  (searcherExecutor-8639-thread-1-processing-n:127.0.0.1:62879_solr x:dv_coll_shard3_replica_n4 s:shard3 c:dv_coll r:core_node7) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.c.SolrCore [dv_coll_shard3_replica_n4] Registered new searcher Searcher@4ba24c32[dv_coll_shard3_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2397519 INFO  (qtp195727981-18617) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica_n4]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:62879/solr/dv_coll_shard1_replica_n1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 3
   [junit4]   2> 2397520 INFO  (qtp2126010706-18612) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1580111836144992256,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2397520 INFO  (qtp2126010706-18612) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 2397520 INFO  (qtp2126010706-18612) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@3c6b6c7e[dv_coll_shard2_replica_n2] main]
   [junit4]   2> 2397520 INFO  (qtp2126010706-18612) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2397525 INFO  (searcherExecutor-8637-thread-1-processing-n:127.0.0.1:48156_solr x:dv_coll_shard2_replica_n2 s:shard2 c:dv_coll r:core_node5) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.c.SolrCore [dv_coll_shard2_replica_n2] Registered new searcher Searcher@3c6b6c7e[dv_coll_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2397525 INFO  (qtp2126010706-18612) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:62879/solr/dv_coll_shard1_replica_n1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 5
   [junit4]   2> 2397526 INFO  (qtp195727981-18620) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=dv_coll:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 12
   [junit4]   2> 2397531 INFO  (qtp195727981-18623) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=dv_coll:4&wt=javabin&version=2}{add=[1 (1580111836154429440)]} 0 1
   [junit4]   2> 2397532 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1580111836157575168,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2397532 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 2397532 INFO  (qtp2126010706-18610) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1580111836157575168,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2397532 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2397532 INFO  (qtp2126010706-18610) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 2397533 INFO  (qtp2126010706-18613) [n:127.0.0.1:48156_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica_n6]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:62879/solr/dv_coll_shard1_replica_n1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
   [junit4]   2> 2397533 INFO  (qtp2126010706-18610) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2397533 INFO  (qtp195727981-18616) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1580111836158623744,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2397533 INFO  (qtp195727981-18616) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@47d237df commitCommandVersion:1580111836158623744
   [junit4]   2> 2397533 INFO  (qtp2126010706-18610) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:62879/solr/dv_coll_shard1_replica_n1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
   [junit4]   2> 2397537 INFO  (qtp195727981-18619) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1580111836162818048,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2397537 INFO  (qtp195727981-18619) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 2397609 INFO  (qtp195727981-18619) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2397609 INFO  (qtp195727981-18619) [n:127.0.0.1:62879_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica_n4]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:62879/solr/dv_coll_shard1_replica_n1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 76
   [junit4]   2> 2397551 INFO  (qtp195727981-18616) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3bf439b4[dv_coll_shard1_replica_n1] main]
   [junit4]   2> 2397610 INFO  (qtp195727981-18616) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2397612 INFO  (searcherExecutor-8657-thread-1-processing-n:127.0.0.1:62879_solr x:dv_coll_shard1_replica_n1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.c.SolrCore [dv_coll_shard1_replica_n1] Registered new searcher Searcher@3bf439b4[dv_coll_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C1)))}
   [junit4]   2> 2397612 INFO  (qtp195727981-18616) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica_n1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:62879/solr/dv_coll_shard1_replica_n1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 80
   [junit4]   2> 2397613 INFO  (qtp195727981-18622) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=dv_coll:4&commit=true&wt=javabin&version=2}{commit=} 0 81
   [junit4]   2> 2397617 WARN  (qtp2126010706-18614) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.r.SimpleFacets Raising facet.mincount from 0 to 1, because field intField is Points-based.
   [junit4]   2> 2397617 WARN  (qtp2126010706-18614) [n:127.0.0.1:48156_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2] o.a.s.r.SimpleFacets Raising facet.mincount from 0 to 1, because field longField is Points-based.
   [junit4]   2> 2397621 WARN  (qtp195727981-18617) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.r.SimpleFacets Raising facet.mincount from 0 to 1, because field intField is Points-based.
   [junit4]   2> 2397622 WARN  (qtp195727981-18617) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1] o.a.s.r.SimpleFacets Raising facet.mincount from 0 to 1, because field longField is Points-based.
   [junit4]   2> 2397622 WARN  (qtp195727981-18617) [n:127.0.0.1:62879_solr c:dv_coll s:shard1 r:cor

[...truncated too long message...]

riter, TermsComponentTest, GraphQueryTest, TestRebalanceLeaders, DocValuesMultiTest, BufferStoreTest, TestNamedUpdateProcessors, TestLeaderInitiatedRecoveryThread, TestMacroExpander, BasicFunctionalityTest, TestApiFramework, AnalyticsQueryTest, DocumentBuilderTest, ConvertedLegacyTest, LeaderFailoverAfterPartitionTest, TestInfoStreamLogging, HdfsRecoveryZkTest, TestComponentsName, SegmentsInfoRequestHandlerTest, TestExclusionRuleCollectionAccess, TestLRUCache, DistributedIntervalFacetingTest, OverriddenZkACLAndCredentialsProvidersTest, HdfsLockFactoryTest, AtomicUpdateProcessorFactoryTest, TestReqParamsAPI, CdcrVersionReplicationTest, TestNumericTerms64, ParsingFieldUpdateProcessorsTest, ShufflingReplicaListTransformerTest, TestSystemCollAutoCreate, TestSolrDeletionPolicy1, TestSchemalessBufferedUpdates, AsyncCallRequestStatusResponseTest, DistributedQueryComponentCustomSortTest, TestSerializedLuceneMatchVersion, TestManagedResourceStorage, BooleanFieldTest, JsonLoaderTest, PluginInfoTest, SubstringBytesRefFilterTest, SignatureUpdateProcessorFactoryTest, TestSolrCloudWithDelegationTokens, TestSchemaNameResource, TestShardHandlerFactory, DocValuesMissingTest, SpellPossibilityIteratorTest, TestPullReplica, TestNonDefinedSimilarityFactory, QueryResultKeyTest, TestLegacyField, SchemaVersionSpecificBehaviorTest, ExternalFileFieldSortTest, TestTlogReplica, TestFastWriter, PropertiesRequestHandlerTest, PreAnalyzedFieldTest, TestBM25SimilarityFactory, TestReload, LeaderElectionIntegrationTest, ShardRoutingTest, ClusterStateUpdateTest, TestRandomFaceting, LeaderElectionTest, ShardRoutingCustomTest, AutoCommitTest, TestDistributedGrouping, TestRecovery, TestHashPartitioner, TestStressReorder, HardAutoCommitTest, TestCoreContainer, SolrCmdDistributorTest, PeerSyncTest, TestBadConfig, SoftAutoCommitTest, PolyFieldTest, WordBreakSolrSpellCheckerTest, SolrCoreCheckLockOnStartupTest, TestAtomicUpdateErrorCases, StatelessScriptUpdateProcessorFactoryTest, IndexBasedSpellCheckerTest, TestQueryUtils, TestWriterPerf, TestReversedWildcardFilterFactory, PrimitiveFieldTypeTest, RequestHandlersTest, TestIndexingPerformance, FieldAnalysisRequestHandlerTest, ReturnFieldsTest, TestAnalyzedSuggestions, BinaryUpdateRequestHandlerTest, PingRequestHandlerTest, HighlighterConfigTest, TestQuerySenderListener, TestSolrIndexConfig, CopyFieldTest, TestStressRecovery, ScriptEngineTest, DateMathParserTest, SpellingQueryConverterTest, RAMDirectoryFactoryTest, TestUtils, ZkNodePropsTest, SystemInfoHandlerTest, SolrTestCaseJ4Test, TestDistributedMissingSort, ActionThrottleTest, AssignBackwardCompatibilityTest, CdcrBootstrapTest, CloudExitableDirectoryReaderTest, CollectionReloadTest, ConcurrentDeleteAndCreateCollectionTest, ConfigSetsAPITest, ConnectionManagerTest, CreateCollectionCleanupTest, CustomCollectionTest, DeleteReplicaTest, DeleteStatusTest, DistribDocExpirationUpdateProcessorTest, DocValuesNotIndexedTest, LeaderElectionContextKeyTest, LeaderFailureAfterFreshStartTest, MigrateRouteKeyTest, MoveReplicaHDFSUlogDirTest, MultiThreadedOCPTest, OverseerCollectionConfigSetProcessorTest, OverseerStatusTest, PeerSyncReplicationTest, ReplicationFactorTest, RestartWhileUpdatingTest, RollingRestartTest, SolrXmlInZkTest, TestAuthenticationFramework, TestCloudPseudoReturnFields, TestClusterProperties, TestCollectionAPI, TestCollectionsAPIViaSolrCloudCluster, TestDeleteCollectionOnDownNodes, TestLockTree, TestPrepRecovery, TestRequestForwarding, TestSSLRandomization, TestSegmentSorting, TestSolrCloudWithSecureImpersonation]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=B44C3FCFFA90B617 -Dtests.slow=true -Dtests.locale=ms-MY -Dtests.timezone=America/Indiana/Vevay -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | TestSolrCloudWithSecureImpersonation (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   [junit4]    >    1) Thread[id=23031, name=jetty-launcher-3705-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]    >    2) Thread[id=23041, name=jetty-launcher-3705-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B44C3FCFFA90B617]:0)
   [junit4] Completed [565/734 (2!)] on J1 in 45.78s, 12 tests, 1 error <<< FAILURES!

[...truncated 37788 lines...]
-documentation-lint:
     [echo] checking for broken html...
    [jtidy] Checking for broken html (such as invalid tags)...
   [delete] Deleting directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/build/jtidy_tmp
     [echo] Checking for broken links...
     [exec] 
     [exec] Crawl/parse...
     [exec] 
     [exec] Verify...
     [exec] 
     [exec] file:///build/docs/sandbox/org/apache/lucene/document/FloatPointNearestNeighbor.html
     [exec]   BROKEN LINK: file:///build/docs/core/org/apache/lucene/document/FloatPointNearestNeighbor.NearestHit.html
     [exec]   BROKEN LINK: file:///build/docs/core/org/apache/lucene/document/FloatPointNearestNeighbor.NearestHit.html
     [exec] 
     [exec] Broken javadocs links were found! Common root causes:
     [exec] * A typo of some sort for manually created links.
     [exec] * Public methods referencing non-public classes in their signature.

BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:826: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:101: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/build.xml:142: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/build.xml:155: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/common-build.xml:2570: exec returned: 1

Total time: 89 minutes 27 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1447 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1447/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC

2 tests failed.
FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI

Error Message:
Error from server at https://127.0.0.1:48207/solr/awhollynewcollection_0_shard1_replica_n1: ClusterState says we are the leader (https://127.0.0.1:48207/solr/awhollynewcollection_0_shard1_replica_n1), but locally we don't think so. Request came from null

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at https://127.0.0.1:48207/solr/awhollynewcollection_0_shard1_replica_n1: ClusterState says we are the leader (https://127.0.0.1:48207/solr/awhollynewcollection_0_shard1_replica_n1), but locally we don't think so. Request came from null
	at __randomizedtesting.SeedInfo.seed([9E12AA94B7B14D2B:D667DE20B18262BE]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:539)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:993)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:459)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:48207/solr/awhollynewcollection_0_shard1_replica_n1: ClusterState says we are the leader (https://127.0.0.1:48207/solr/awhollynewcollection_0_shard1_replica_n1), but locally we don't think so. Request came from null
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:516)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more


FAILED:  junit.framework.TestSuite.org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth:     1) Thread[id=36300, name=jetty-launcher-6753-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)    2) Thread[id=36296, name=jetty-launcher-6753-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth: 
   1) Thread[id=36300, name=jetty-launcher-6753-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   2) Thread[id=36296, name=jetty-launcher-6753-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
	at __randomizedtesting.SeedInfo.seed([9E12AA94B7B14D2B]:0)




Build Log:
[...truncated 12647 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest_9E12AA94B7B14D2B-001/init-core-data-001
   [junit4]   2> 1784614 WARN  (SUITE-CollectionsAPIDistributedZkTest-seed#[9E12AA94B7B14D2B]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=32 numCloses=32
   [junit4]   2> 1784614 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[9E12AA94B7B14D2B]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1784616 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[9E12AA94B7B14D2B]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1784617 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[9E12AA94B7B14D2B]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1784618 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[9E12AA94B7B14D2B]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest_9E12AA94B7B14D2B-001/tempDir-001
   [junit4]   2> 1784618 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[9E12AA94B7B14D2B]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1784618 INFO  (Thread-3646) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1784618 INFO  (Thread-3646) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1784628 ERROR (Thread-3646) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1784718 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[9E12AA94B7B14D2B]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:43924
   [junit4]   2> 1784761 INFO  (jetty-launcher-3041-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1784761 INFO  (jetty-launcher-3041-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1784761 INFO  (jetty-launcher-3041-thread-4) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1784763 INFO  (jetty-launcher-3041-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@54e02155{/solr,null,AVAILABLE}
   [junit4]   2> 1784770 INFO  (jetty-launcher-3041-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7ca7c67c{/solr,null,AVAILABLE}
   [junit4]   2> 1784770 INFO  (jetty-launcher-3041-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3da27b85{SSL,[ssl, http/1.1]}{127.0.0.1:40063}
   [junit4]   2> 1784770 INFO  (jetty-launcher-3041-thread-2) [    ] o.e.j.s.Server Started @1787077ms
   [junit4]   2> 1784770 INFO  (jetty-launcher-3041-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40063}
   [junit4]   2> 1784770 INFO  (jetty-launcher-3041-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6ee629c1{SSL,[ssl, http/1.1]}{127.0.0.1:39795}
   [junit4]   2> 1784770 INFO  (jetty-launcher-3041-thread-1) [    ] o.e.j.s.Server Started @1787078ms
   [junit4]   2> 1784771 INFO  (jetty-launcher-3041-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39795}
   [junit4]   2> 1784771 ERROR (jetty-launcher-3041-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1784771 ERROR (jetty-launcher-3041-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1784772 INFO  (jetty-launcher-3041-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1784772 INFO  (jetty-launcher-3041-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1784772 INFO  (jetty-launcher-3041-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1784772 INFO  (jetty-launcher-3041-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-01T13:24:22.560Z
   [junit4]   2> 1784772 INFO  (jetty-launcher-3041-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6b24cffe{/solr,null,AVAILABLE}
   [junit4]   2> 1784773 INFO  (jetty-launcher-3041-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5959379b{SSL,[ssl, http/1.1]}{127.0.0.1:48207}
   [junit4]   2> 1784773 INFO  (jetty-launcher-3041-thread-4) [    ] o.e.j.s.Server Started @1787081ms
   [junit4]   2> 1784774 INFO  (jetty-launcher-3041-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=48207}
   [junit4]   2> 1784774 ERROR (jetty-launcher-3041-thread-4) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1784774 INFO  (jetty-launcher-3041-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1784774 INFO  (jetty-launcher-3041-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1784774 INFO  (jetty-launcher-3041-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1784774 INFO  (jetty-launcher-3041-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-01T13:24:22.562Z
   [junit4]   2> 1784776 INFO  (jetty-launcher-3041-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1784776 INFO  (jetty-launcher-3041-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1784777 INFO  (jetty-launcher-3041-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1784777 INFO  (jetty-launcher-3041-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-01T13:24:22.565Z
   [junit4]   2> 1784784 INFO  (jetty-launcher-3041-thread-3) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1784785 INFO  (jetty-launcher-3041-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1784786 INFO  (jetty-launcher-3041-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1784786 INFO  (jetty-launcher-3041-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@770e915e{/solr,null,AVAILABLE}
   [junit4]   2> 1784787 INFO  (jetty-launcher-3041-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@71ee98a4{SSL,[ssl, http/1.1]}{127.0.0.1:63272}
   [junit4]   2> 1784787 INFO  (jetty-launcher-3041-thread-3) [    ] o.e.j.s.Server Started @1787094ms
   [junit4]   2> 1784787 INFO  (jetty-launcher-3041-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=63272}
   [junit4]   2> 1784787 ERROR (jetty-launcher-3041-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1784787 INFO  (jetty-launcher-3041-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1784787 INFO  (jetty-launcher-3041-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1784787 INFO  (jetty-launcher-3041-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1784787 INFO  (jetty-launcher-3041-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-01T13:24:22.575Z
   [junit4]   2> 1784804 INFO  (jetty-launcher-3041-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1784806 INFO  (jetty-launcher-3041-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1784818 INFO  (jetty-launcher-3041-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43924/solr
   [junit4]   2> 1784822 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15ed81bed0d0004, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1784824 INFO  (jetty-launcher-3041-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43924/solr
   [junit4]   2> 1784824 INFO  (jetty-launcher-3041-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43924/solr
   [junit4]   2> 1784858 INFO  (jetty-launcher-3041-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43924/solr
   [junit4]   2> 1784887 INFO  (jetty-launcher-3041-thread-3) [n:127.0.0.1:63272_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1784887 INFO  (jetty-launcher-3041-thread-3) [n:127.0.0.1:63272_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:63272_solr
   [junit4]   2> 1784887 INFO  (jetty-launcher-3041-thread-1) [n:127.0.0.1:39795_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1784889 INFO  (jetty-launcher-3041-thread-1) [n:127.0.0.1:39795_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39795_solr
   [junit4]   2> 1784889 INFO  (jetty-launcher-3041-thread-3) [n:127.0.0.1:63272_solr    ] o.a.s.c.Overseer Overseer (id=98753856301498378-127.0.0.1:63272_solr-n_0000000000) starting
   [junit4]   2> 1784929 INFO  (jetty-launcher-3041-thread-2) [n:127.0.0.1:40063_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1784932 INFO  (zkCallback-3058-thread-1-processing-n:127.0.0.1:63272_solr) [n:127.0.0.1:63272_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1784936 INFO  (jetty-launcher-3041-thread-2) [n:127.0.0.1:40063_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40063_solr
   [junit4]   2> 1784942 INFO  (zkCallback-3063-thread-1-processing-n:127.0.0.1:40063_solr) [n:127.0.0.1:40063_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1784945 INFO  (zkCallback-3063-thread-1-processing-n:127.0.0.1:40063_solr) [n:127.0.0.1:40063_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1784972 INFO  (zkCallback-3062-thread-1-processing-n:127.0.0.1:39795_solr) [n:127.0.0.1:39795_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1784978 INFO  (OverseerStateUpdate-98753856301498378-127.0.0.1:63272_solr-n_0000000000) [n:127.0.0.1:63272_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1785001 INFO  (jetty-launcher-3041-thread-2) [n:127.0.0.1:40063_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4ea2b1e7
   [junit4]   2> 1785017 INFO  (jetty-launcher-3041-thread-1) [n:127.0.0.1:39795_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4ea2b1e7
   [junit4]   2> 1785019 INFO  (jetty-launcher-3041-thread-3) [n:127.0.0.1:63272_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63272_solr
   [junit4]   2> 1785019 INFO  (jetty-launcher-3041-thread-2) [n:127.0.0.1:40063_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4ea2b1e7
   [junit4]   2> 1785019 INFO  (jetty-launcher-3041-thread-2) [n:127.0.0.1:40063_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4ea2b1e7
   [junit4]   2> 1785020 INFO  (jetty-launcher-3041-thread-2) [n:127.0.0.1:40063_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest_9E12AA94B7B14D2B-001/tempDir-001/node2
   [junit4]   2> 1785041 INFO  (zkCallback-3063-thread-1-processing-n:127.0.0.1:40063_solr) [n:127.0.0.1:40063_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1785042 INFO  (zkCallback-3062-thread-1-processing-n:127.0.0.1:39795_solr) [n:127.0.0.1:39795_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1785045 INFO  (zkCallback-3058-thread-1-processing-n:127.0.0.1:63272_solr) [n:127.0.0.1:63272_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1785047 INFO  (jetty-launcher-3041-thread-1) [n:127.0.0.1:39795_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4ea2b1e7
   [junit4]   2> 1785047 INFO  (jetty-launcher-3041-thread-1) [n:127.0.0.1:39795_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4ea2b1e7
   [junit4]   2> 1785048 INFO  (jetty-launcher-3041-thread-1) [n:127.0.0.1:39795_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest_9E12AA94B7B14D2B-001/tempDir-001/node1
   [junit4]   2> 1785092 INFO  (jetty-launcher-3041-thread-4) [n:127.0.0.1:48207_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 1785095 INFO  (jetty-launcher-3041-thread-4) [n:127.0.0.1:48207_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1785101 INFO  (jetty-launcher-3041-thread-4) [n:127.0.0.1:48207_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48207_solr
   [junit4]   2> 1785103 INFO  (zkCallback-3062-thread-1-processing-n:127.0.0.1:39795_solr) [n:127.0.0.1:39795_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1785104 INFO  (zkCallback-3063-thread-1-processing-n:127.0.0.1:40063_solr) [n:127.0.0.1:40063_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1785104 INFO  (zkCallback-3058-thread-2-processing-n:127.0.0.1:63272_solr) [n:127.0.0.1:63272_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1785107 INFO  (zkCallback-3051-thread-1-processing-n:127.0.0.1:48207_solr) [n:127.0.0.1:48207_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1785141 INFO  (jetty-launcher-3041-thread-3) [n:127.0.0.1:63272_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4ea2b1e7
   [junit4]   2> 1785148 INFO  (jetty-launcher-3041-thread-3) [n:127.0.0.1:63272_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4ea2b1e7
   [junit4]   2> 1785148 INFO  (jetty-launcher-3041-thread-3) [n:127.0.0.1:63272_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4ea2b1e7
   [junit4]   2> 1785149 INFO  (jetty-launcher-3041-thread-3) [n:127.0.0.1:63272_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest_9E12AA94B7B14D2B-001/tempDir-001/node4
   [junit4]   2> 1785223 INFO  (jetty-launcher-3041-thread-4) [n:127.0.0.1:48207_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4ea2b1e7
   [junit4]   2> 1785229 INFO  (jetty-launcher-3041-thread-4) [n:127.0.0.1:48207_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4ea2b1e7
   [junit4]   2> 1785229 INFO  (jetty-launcher-3041-thread-4) [n:127.0.0.1:48207_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4ea2b1e7
   [junit4]   2> 1785230 INFO  (jetty-launcher-3041-thread-4) [n:127.0.0.1:48207_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest_9E12AA94B7B14D2B-001/tempDir-001/node3
   [junit4]   2> 1785257 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[9E12AA94B7B14D2B]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1785258 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[9E12AA94B7B14D2B]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43924/solr ready
   [junit4]   2> 1785461 INFO  (TEST-CollectionsAPIDistributedZkTest.testZeroNumShards-seed#[9E12AA94B7B14D2B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testZeroNumShards
   [junit4]   2> 1785463 INFO  (TEST-CollectionsAPIDistributedZkTest.testZeroNumShards-seed#[9E12AA94B7B14D2B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1785513 INFO  (qtp716286851-18523) [n:127.0.0.1:48207_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=10&collection.configName=conf&name=acollection&action=CREATE&numShards=0&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1785516 INFO  (OverseerThreadFactory-6828-thread-1-processing-n:127.0.0.1:63272_solr) [n:127.0.0.1:63272_solr    ] o.a.s.c.CreateCollectionCmd Create collection acollection
   [junit4]   2> 1785517 ERROR (OverseerThreadFactory-6828-thread-1-processing-n:127.0.0.1:63272_solr) [n:127.0.0.1:63272_solr    ] o.a.s.c.OverseerCollectionMessageHandler Collection: acollection operation: create failed:org.apache.solr.common.SolrException: numShards must be > 0
   [junit4]   2> 	at org.apache.solr.cloud.CreateCollectionCmd.call(CreateCollectionCmd.java:132)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:228)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:464)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 1785534 INFO  (qtp716286851-18523) [n:127.0.0.1:48207_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=10&collection.configName=conf&name=acollection&action=CREATE&numShards=0&wt=javabin&version=2} status=400 QTime=21
   [junit4]   2> 1785534 INFO  (TEST-CollectionsAPIDistributedZkTest.testZeroNumShards-seed#[9E12AA94B7B14D2B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testZeroNumShards
   [junit4]   2> 1785731 INFO  (TEST-CollectionsAPIDistributedZkTest.testCreateShouldFailOnExistingCore-seed#[9E12AA94B7B14D2B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testCreateShouldFailOnExistingCore
   [junit4]   2> 1785733 INFO  (TEST-CollectionsAPIDistributedZkTest.testCreateShouldFailOnExistingCore-seed#[9E12AA94B7B14D2B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1785738 INFO  (qtp716286851-18519) [n:127.0.0.1:48207_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=halfcollectionblocker&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1785743 INFO  (OverseerThreadFactory-6828-thread-2-processing-n:127.0.0.1:63272_solr) [n:127.0.0.1:63272_solr    ] o.a.s.c.CreateCollectionCmd Create collection halfcollectionblocker
   [junit4]   2> 1785745 INFO  (OverseerCollectionConfigSetProcessor-98753856301498378-127.0.0.1:63272_solr-n_0000000000) [n:127.0.0.1:63272_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1785745 WARN  (OverseerThreadFactory-6828-thread-2-processing-n:127.0.0.1:63272_solr) [n:127.0.0.1:63272_solr    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (halfcollectionblocker) without cores.
   [junit4]   2> 1785869 INFO  (qtp716286851-18519) [n:127.0.0.1:48207_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1785870 INFO  (qtp716286851-18519) [n:127.0.0.1:48207_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=halfcollectionblocker&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&wt=javabin&version=2} status=0 QTime=131
   [junit4]   2> 1785871 INFO  (qtp716286851-18520) [n:127.0.0.1:48207_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:40063_solr&name=halfcollection_shard1_replica_n1&action=ADDREPLICA&collection=halfcollectionblocker&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1785874 INFO  (OverseerCollectionConfigSetProcessor-98753856301498378-127.0.0.1:63272_solr-n_0000000000) [n:127.0.0.1:63272_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1785875 INFO  (OverseerThreadFactory-6828-thread-3-processing-n:127.0.0.1:63272_solr) [n:127.0.0.1:63272_solr    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:40063_solr for creating new replica
   [junit4]   2> 1785877 INFO  (OverseerStateUpdate-98753856301498378-127.0.0.1:63272_solr-n_0000000000) [n:127.0.0.1:63272_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"addreplica",
   [junit4]   2>   "collection":"halfcollectionblocker",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"halfcollection_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:40063/solr",
   [junit4]   2>   "node_name":"127.0.0.1:40063_solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1786152 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node21&collection.configName=conf&name=halfcollection_shard1_replica_n1&action=CREATE&collection=halfcollectionblocker&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1786153 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1786175 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1786182 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.s.IndexSchema [halfcollection_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 1786184 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1786184 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'halfcollection_shard1_replica_n1' using configuration from collection halfcollectionblocker, trusted=true
   [junit4]   2> 1786185 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.halfcollectionblocker.shard1.core_node21' (registry 'solr.core.halfcollectionblocker.shard1.core_node21') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4ea2b1e7
   [junit4]   2> 1786185 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1786185 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.SolrCore [[halfcollection_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest_9E12AA94B7B14D2B-001/tempDir-001/node2/halfcollection_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest_9E12AA94B7B14D2B-001/tempDir-001/node2/halfcollection_shard1_replica_n1/data/]
   [junit4]   2> 1786250 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1786250 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1786251 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1786251 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1786252 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5159c75a[halfcollection_shard1_replica_n1] main]
   [junit4]   2> 1786254 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1786254 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1786255 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1786256 INFO  (searcherExecutor-6837-thread-1-processing-n:127.0.0.1:40063_solr x:halfcollection_shard1_replica_n1 s:shard1 c:halfcollectionblocker r:core_node21) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.SolrCore [halfcollection_shard1_replica_n1] Registered new searcher Searcher@5159c75a[halfcollection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1786256 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580061702534201344
   [junit4]   2> 1786263 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1786263 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1786263 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:40063/solr/halfcollection_shard1_replica_n1/
   [junit4]   2> 1786263 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1786263 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:40063/solr/halfcollection_shard1_replica_n1/ has no replicas
   [junit4]   2> 1786263 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1786268 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:40063/solr/halfcollection_shard1_replica_n1/ shard1
   [junit4]   2> 1786377 INFO  (zkCallback-3063-thread-1-processing-n:127.0.0.1:40063_solr) [n:127.0.0.1:40063_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfcollectionblocker/state.json] for collection [halfcollectionblocker] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1786378 INFO  (zkCallback-3063-thread-2-processing-n:127.0.0.1:40063_solr) [n:127.0.0.1:40063_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfcollectionblocker/state.json] for collection [halfcollectionblocker] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1786426 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1786429 INFO  (qtp453303865-18512) [n:127.0.0.1:40063_solr c:halfcollectionblocker s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node21&collection.configName=conf&name=halfcollection_shard1_replica_n1&action=CREATE&collection=halfcollectionblocker&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=277
   [junit4]   2> 1786435 INFO  (qtp716286851-18520) [n:127.0.0.1:48207_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:40063_solr&name=halfcollection_shard1_replica_n1&action=ADDREPLICA&collection=halfcollectionblocker&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=564
   [junit4]   2> 1786436 INFO  (qtp716286851-18525) [n:127.0.0.1:48207_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=halfcollectionblocker2&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1786443 INFO  (OverseerThreadFactory-6828-thread-4-processing-n:127.0.0.1:63272_solr) [n:127.0.0.1:63272_solr    ] o.a.s.c.CreateCollectionCmd Create collection halfcollectionblocker2
   [junit4]   2> 1786444 INFO  (OverseerCollectionConfigSetProcessor-98753856301498378-127.0.0.1:63272_solr-n_0000000000) [n:127.0.0.1:63272_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1786444 WARN  (OverseerThreadFactory-6828-thread-4-processing-n:127.0.0.1:63272_solr) [n:127.0.0.1:63272_solr    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (halfcollectionblocker2) without cores.
   [junit4]   2> 1786563 INFO  (zkCallback-3063-thread-1-processing-n:127.0.0.1:40063_solr) [n:127.0.0.1:40063_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfcollectionblocker/state.json] for collection [halfcollectionblocker] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1786563 INFO  (zkCallback-3063-thread-2-processing-n:127.0.0.1:40063_solr) [n:127.0.0.1:40063_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfcollectionblocker/state.json] for collection [halfcollectionblocker] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1786659 INFO  (qtp716286851-18525) [n:127.0.0.1:48207_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1786660 INFO  (qtp716286851-18525) [n:127.0.0.1:48207_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=halfcollectionblocker2&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&wt=javabin&version=2} status=0 QTime=223
   [junit4]   2> 1786661 INFO  (qtp716286851-18521) [n:127.0.0.1:48207_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:39795_solr&name=halfcollection_shard1_replica_n1&action=ADDREPLICA&collection=halfcollectionblocker2&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1786668 INFO  (OverseerCollectionConfigSetProcessor-98753856301498378-127.0.0.1:63272_solr-n_0000000000) [n:127.0.0.1:63272_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1786669 INFO  (OverseerThreadFactory-6828-thread-5-processing-n:127.0.0.1:63272_solr) [n:127.0.0.1:63272_solr    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:39795_solr for creating new replica
   [junit4]   2> 1786674 INFO  (OverseerStateUpdate-98753856301498378-127.0.0.1:63272_solr-n_0000000000) [n:127.0.0.1:63272_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"addreplica",
   [junit4]   2>   "collection":"halfcollectionblocker2",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"halfcollection_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:39795/solr",
   [junit4]   2>   "node_name":"127.0.0.1:39795_solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1786935 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node21&collection.configName=conf&name=halfcollection_shard1_replica_n1&action=CREATE&collection=halfcollectionblocker2&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1786935 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1786956 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1786963 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.s.IndexSchema [halfcollection_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 1786966 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1786966 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'halfcollection_shard1_replica_n1' using configuration from collection halfcollectionblocker2, trusted=true
   [junit4]   2> 1786966 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.halfcollectionblocker2.shard1.core_node21' (registry 'solr.core.halfcollectionblocker2.shard1.core_node21') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4ea2b1e7
   [junit4]   2> 1786966 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1786967 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.SolrCore [[halfcollection_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest_9E12AA94B7B14D2B-001/tempDir-001/node1/halfcollection_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest_9E12AA94B7B14D2B-001/tempDir-001/node1/halfcollection_shard1_replica_n1/data/]
   [junit4]   2> 1787017 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1787017 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1787031 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1787031 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1787032 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3631dfeb[halfcollection_shard1_replica_n1] main]
   [junit4]   2> 1787035 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1787036 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1787036 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1787037 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580061703353139200
   [junit4]   2> 1787040 INFO  (searcherExecutor-6842-thread-1-processing-n:127.0.0.1:39795_solr x:halfcollection_shard1_replica_n1 s:shard1 c:halfcollectionblocker2 r:core_node21) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.SolrCore [halfcollection_shard1_replica_n1] Registered new searcher Searcher@3631dfeb[halfcollection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1787045 INFO  (zkCallback-3062-thread-1-processing-n:127.0.0.1:39795_solr) [n:127.0.0.1:39795_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfcollectionblocker2/state.json] for collection [halfcollectionblocker2] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1787046 INFO  (zkCallback-3062-thread-2-processing-n:127.0.0.1:39795_solr) [n:127.0.0.1:39795_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfcollectionblocker2/state.json] for collection [halfcollectionblocker2] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1787049 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1787049 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1787049 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:39795/solr/halfcollection_shard1_replica_n1/
   [junit4]   2> 1787049 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1787050 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:39795/solr/halfcollection_shard1_replica_n1/ has no replicas
   [junit4]   2> 1787050 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1787054 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:39795/solr/halfcollection_shard1_replica_n1/ shard1
   [junit4]   2> 1787158 INFO  (zkCallback-3062-thread-1-processing-n:127.0.0.1:39795_solr) [n:127.0.0.1:39795_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfcollectionblocker2/state.json] for collection [halfcollectionblocker2] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1787158 INFO  (zkCallback-3062-thread-2-processing-n:127.0.0.1:39795_solr) [n:127.0.0.1:39795_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfcollectionblocker2/state.json] for collection [halfcollectionblocker2] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1787213 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1787219 INFO  (qtp1452980072-18531) [n:127.0.0.1:39795_solr c:halfcollectionblocker2 s:shard1 r:core_node21 x:halfcollection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node21&collection.configName=conf&name=halfcollection_shard1_replica_n1&action=CREATE&collection=halfcollectionblocker2&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=284
   [junit4]   2> 1787222 INFO  (qtp716286851-18521) [n:127.0.0.1:48207_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:39795_solr&name=halfcollection_shard1_replica_n1&action=ADDREPLICA&collection=halfcollectionblocker2&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=561
   [junit4]   2> 1787223 INFO  (qtp716286851-18524) [n:127.0.0.1:48207_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=halfcollection&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=127.0.0.1:40063_solr,127.0.0.1:39795_solr&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1787226 INFO  (OverseerThreadFactory-6828-thread-5-processing-n:127.0.0.1:63272_solr) [n:127.0.0.1:63272_solr    ] o.a.s.c.CreateCollectionCmd Create collection halfcollection
   [junit4]   2> 1787227 INFO  (OverseerCollectionConfigSetProcessor-98753856301498378-127.0.0.1:63272_solr-n_0000000000) [n:127.0.0.1:63272_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000008 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1787337 INFO  (zkCallback-3062-thread-2-processing-n:127.0.0.1:39795_solr) [n:127.0.0.1:39795_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfcollectionblocker2/state.json] for collection [halfcollectionblocker2] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1787337 INFO  (zkCallback-3062-thread-1-processing-n:127.0.0.1:39795_solr) [n:127.0.0.1:39795_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfcollectionblocker2/state.json] for collection [halfcollectionblocker2] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1787448 INFO  (OverseerStateUpdate-98753856301498378-127.0.0.1:63272_solr-n_0000000000) [n:127.0.0.1:63272_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"halfcollection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"halfcollection_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:40063/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1787456 INFO  (OverseerStateUpdate-98753856301498378-127.0.0.1:63272_solr-n_0000000000) [n:127.0.0.1:63272_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"halfcollection",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"halfcollection_shard2_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:39795/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1787668 INFO  (qtp453303865-18515) [n:127.0.0.1:40063_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=halfcollection_shard1_replica_n1&action=CREATE&numShards=2&collection=halfcollection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1787668 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=halfcollection_shard2_replica_n2&action=CREATE&numShards=2&collection=halfcollection&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1787669 WARN  (qtp453303865-18515) [n:127.0.0.1:40063_solr    ] o.a.s.c.CoreContainer Creating a core with existing name is not allowed
   [junit4]   2> 1787669 ERROR (qtp453303865-18515) [n:127.0.0.1:40063_solr    ] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Core with name 'halfcollection_shard1_replica_n1' already exists.
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:920)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 1787670 INFO  (qtp453303865-18515) [n:127.0.0.1:40063_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=halfcollection_shard1_replica_n1&action=CREATE&numShards=2&collection=halfcollection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=500 QTime=11
   [junit4]   2> 1787671 ERROR (qtp453303865-18515) [n:127.0.0.1:40063_solr    ] o.a.s.s.HttpSolrCall null:org.apache.solr.common.SolrException: Core with name 'halfcollection_shard1_replica_n1' already exists.
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:920)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 1787672 ERROR (OverseerThreadFactory-6828-thread-5-processing-n:127.0.0.1:63272_solr) [n:127.0.0.1:63272_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: https://127.0.0.1:40063/solr
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:40063/solr: Core with name 'halfcollection_shard1_replica_n1' already exists.
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1787785 INFO  (zkCallback-3062-thread-1-processing-n:127.0.0.1:39795_solr) [n:127.0.0.1:39795_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfcollection/state.json] for collection [halfcollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1787785 INFO  (zkCallback-3062-thread-2-processing-n:127.0.0.1:39795_solr) [n:127.0.0.1:39795_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfcollection/state.json] for collection [halfcollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1788695 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1788708 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.s.IndexSchema [halfcollection_shard2_replica_n2] Schema name=minimal
   [junit4]   2> 1788712 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1788712 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'halfcollection_shard2_replica_n2' using configuration from collection halfcollection, trusted=true
   [junit4]   2> 1788713 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.halfcollection.shard2.replica_n2' (registry 'solr.core.halfcollection.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4ea2b1e7
   [junit4]   2> 1788713 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1788713 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.c.SolrCore [[halfcollection_shard2_replica_n2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest_9E12AA94B7B14D2B-001/tempDir-001/node1/halfcollection_shard2_replica_n2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest_9E12AA94B7B14D2B-001/tempDir-001/node1/halfcollection_shard2_replica_n2/data/]
   [junit4]   2> 1788771 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1788771 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1788773 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1788773 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1788774 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@15478deb[halfcollection_shard2_replica_n2] main]
   [junit4]   2> 1788776 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1788776 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1788777 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1788777 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580061705177661440
   [junit4]   2> 1788777 INFO  (searcherExecutor-6847-thread-1-processing-n:127.0.0.1:39795_solr x:halfcollection_shard2_replica_n2 s:shard2 c:halfcollection r:core_node4) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.c.SolrCore [halfcollection_shard2_replica_n2] Registered new searcher Searcher@15478deb[halfcollection_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1788784 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1788784 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1788784 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:39795/solr/halfcollection_shard2_replica_n2/
   [junit4]   2> 1788784 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1788784 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.c.SyncStrategy https://127.0.0.1:39795/solr/halfcollection_shard2_replica_n2/ has no replicas
   [junit4]   2> 1788784 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1788789 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:39795/solr/halfcollection_shard2_replica_n2/ shard2
   [junit4]   2> 1788894 INFO  (zkCallback-3062-thread-1-processing-n:127.0.0.1:39795_solr) [n:127.0.0.1:39795_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfcollection/state.json] for collection [halfcollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1788894 INFO  (zkCallback-3062-thread-2-processing-n:127.0.0.1:39795_solr) [n:127.0.0.1:39795_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfcollection/state.json] for collection [halfcollection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1788942 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1788946 INFO  (qtp1452980072-18532) [n:127.0.0.1:39795_solr c:halfcollection s:shard2 r:core_node4 x:halfcollection_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=halfcollection_shard2_replica_n2&action=CREATE&numShards=2&collection=halfcollection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1278
   [junit4]   2> 1788947 ERROR (OverseerThreadFactory-6828-thread-5-processing-n:127.0.0.1:63272_solr) [n:127.0.0.1:63272_solr    ] o.a.s.c.OverseerCollectionMessageHandler Cleaning up collection [halfcollection].
   [junit4]   2> 1788949 INFO  (OverseerThreadFactory-6828-thread-5-processing-n:127.0.0.1:63272_solr) [n:127.0.0.1:63272_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1788952 INFO  (qtp1452980072-18533) [n:127.0.0.1:39795_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.halfcollection.shard2.replica_n2, tag=null
   [junit4]   2> 1788952 INFO  (qtp1452980072-18533) [n:127.0.0.1:39795_solr    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4b75f5e2: rootName = null, domain = solr.core.halfcollection.shard2.replica_n2, service url = null, agent id = null] for registry solr.core.halfcollection.shard2.replica_n2 / com.codahale.metrics.MetricRegistry@5f422e23
   [junit4]   2> 1788969 INFO  (qtp1452980072-18533) [n:127.0.0.1:39795_solr    ] o.a.s.c.SolrCore [halfcollection_shard2_replica_n2]  CLOSING SolrCore org.apache.solr.core.SolrCore@35caed3b
   [junit4]   2> 1788969 INFO  (qtp1452980072-18533) [n:127.0.0.1:39795_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.halfcollection.shard2.replica_n2, tag=902491451
   [junit4]   2> 1788969 INFO  (qtp1452980072-18533) [n:127.0.0.1:39795_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.halfcollection.shard2.leader, tag=902491451
   [junit4]   2> 1788979 INFO  (qtp1452980072-18533) [n:127.0.0.1:39795_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=halfcollection_shard2_replica_n2&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=28
   [junit4]   2> 1789685 INFO  (OverseerThreadFactory-6828-thread-5-processing-n:127.0.0.1:63272_solr) [n:127.0.0.1:63272_solr    ] o.a.s.c.CreateCollectionCmd Cleaned up artifacts for failed create collection for [halfcollection]
   [junit4]   2> 1789690 INFO  (qtp716286851-18524) [n:127.0.0.1:48207_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1791232 INFO  (OverseerCollectionConfigSetProcessor-98753856301498378-127.0.0.1:63272_solr-n_0000000000) [n:127.0.0.1:63272_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000010 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1819731 ERROR (qtp716286851-18524) [n:127.0.0.1:48207_solr    ] o.a.s.h.a.CollectionsHandler Timed out waiting for new collection's replicas to become ACTIVE with timeout=30
   [junit4]   2> 1819731 INFO  (qtp716286851-18524) [n:127.0.0.1:48207_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=halfcollection&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=127.0.0.1:40063_solr,127.0.0.1:39795_solr&wt=javabin&version=2} status=0 QTime=32514
   [junit4]   2> 1819731 INFO  (TEST-CollectionsAPIDistributedZkTest.testCreateShouldFailOnExistingCore-seed#[9E12AA94B7B14D2B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testCreateShouldFailOnExistingCore
   [junit4]   2> 1819973 INFO  (TEST-CollectionsAPIDistributedZkTest.deleteCollectionOnlyInZk-seed#[9E12AA94B7B14D2B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting deleteCollectionOnlyInZk
   [junit4]   2> 1819974 INFO  (TEST-CollectionsAPIDistributedZkTest.deleteCollectionOnlyInZk-seed#[9E12AA94B7B14D2B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1819976 INFO  (qtp716286851-18522) [n:127.0.0.1:48207_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=halfcollectionblocker2&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1819985 INFO  (OverseerThreadFactory-6828-thread-5-processing-n:127.0.0.1:63272_solr) [n:127.0.0.1:63272_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1819989 INFO  (qtp1452980072-18534) [n:127.0.0.1:39795_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.halfcollectionblocker2.shard1.core_node21, tag=null
   [junit4]   2> 1819989 INFO  (qtp1452980072-18534) [n:127.0.0.1:39795_solr    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4239e9a5: rootName = null, domain = solr.core.halfcollectionblocker2.shard1.core_node21, service url = null, agent id = null] for registry solr.core.halfcollectionblocker2.shard1.core_node21 / com.codahale.metrics.MetricRegistry@43724524
   [junit4]   2> 1819999 INFO  (qtp1452980072-18534) [n:127.0.0.1:39795_solr    ] o.a.s.c.SolrCore [halfcollection_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@6f90f09f
   [junit4]   2> 1819999 INFO  (qtp1452980072-18534) [n:127.0.0.1:39795_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.halfcollectionblocker2.shard1.core_node21, tag=1871769759
   [junit4]   2> 1819999 INFO  (qtp1452980072-18534) [n:127.0.0.1:39795_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.halfcollectionblocker2.shard1.leader, tag=1871769759
   [junit4]   2> 1820007 INFO  (qtp1452980072-18534) [n:127.0.0.1:39795_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=halfcollection_shard1_replica_n1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=19
   [junit4]   2> 1820717 INFO  (qtp716286851-18522) [n:127.0.0.1:48207_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=halfcollectionblocker2&action=DELETE&wt=javabin&version=2} status=0 QTime=741
   [junit4]   2> 1820718 INFO  (qtp716286851-18523) [n:127.0.0.1:48207_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=halfcollectionblocker&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1820722 INFO  (OverseerCollectionConfigSetProcessor-98753856301498378-127.0.0.1:63272_solr-n_0000000000) [n:127.0.0.1:63272_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000012 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1820722 INFO  (OverseerThreadFactory-6828-thread-5-processing-n:127.0.0.1:63272_solr) [n:127.0.0.1:63272_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1820729 INFO  (qtp453303865-18518) [n:127.0.0.1:40063_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.halfcollectionblocker.shard1.core_node21, tag=null
   [junit4]   2> 1820729 INFO  (qtp453303865-18518) [n:127.0.0.1:40063_solr    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6814226c: rootName = null, domain = solr.core.halfcollectionblocker.shard1.core_node21, service url = null, agent id = null] for registry solr.core.halfcollectionblocker.shard1.co

[...truncated too long message...]

figSetsAPITest, TestSolrCloudWithSecureImpersonation, SegmentsInfoRequestHandlerTest, DocumentAnalysisRequestHandlerTest, TestCollationField, SolrIndexConfigTest, TestQueryUtils, TestDynamicFieldCollectionResource, TestLMJelinekMercerSimilarityFactory, TestFreeTextSuggestions, JSONWriterTest, TestMultiValuedNumericRangeQuery, TriLevelCompositeIdRoutingTest, TestPKIAuthenticationPlugin, TestNamedUpdateProcessors, SpellCheckCollatorTest, StatelessScriptUpdateProcessorFactoryTest, PeerSyncWithIndexFingerprintCachingTest, TestSerializedLuceneMatchVersion, TestSubQueryTransformer, BasicFunctionalityTest, SuggesterFSTTest, TestGraphMLResponseWriter, TestUseDocValuesAsStored2, OpenCloseCoreStressTest, TestQueryWrapperFilter, ZkSolrClientTest, ManagedSchemaRoundRobinCloudTest, TestLegacyField, TestBulkSchemaAPI, LukeRequestHandlerTest, TestDistributedStatsComponentCardinality, HdfsBasicDistributedZkTest, TestManagedSchemaAPI, TestComplexPhraseLeadingWildcard, SpellCheckCollatorWithCollapseTest, ActionThrottleTest, MBeansHandlerTest, ShowFileRequestHandlerTest, V2StandaloneTest, TestSolrIndexConfig, ShardRoutingCustomTest, TestReload, TestWordDelimiterFilterFactory, PreAnalyzedFieldTest, AssignBackwardCompatibilityTest, TestPartialUpdateDeduplication, PluginInfoTest, TestJettySolrRunner, TestCloudSchemaless, TestIndexSearcher, BasicDistributedZkTest, BasicDistributedZk2Test, OverseerTest, BasicZkTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, TestDistributedGrouping, TestFaceting, TestRecovery, TestHashPartitioner, ZkControllerTest, HardAutoCommitTest, TestGroupingSearch, SoftAutoCommitTest, SearchHandlerTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, TestSearchPerf, TestSolrCoreProperties, TestPostingsSolrHighlighter, DirectSolrConnectionTest, TestLuceneMatchVersion, SpellPossibilityIteratorTest, TestCharFilters, TestCodecSupport, TestXIncludeConfig, TestLMDirichletSimilarityFactory, TestIBSimilarityFactory, TestBM25SimilarityFactory, TestFastWriter, TimeZoneUtilsTest, ResourceLoaderTest, OpenExchangeRatesOrgProviderTest, TestFastLRUCache, ChaosMonkeyNothingIsSafeTest, TestSystemIdResolver, PrimUtilsTest, TestSuggestSpellingConverter, DateFieldTest, ClusterStateTest, TestSolrJ, TestUtils, TestDocumentBuilder, ZkNodePropsTest, SliceStateTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestDistributedMissingSort, TestTolerantSearch, TestLuceneIndexBackCompat, TestEmbeddedSolrServerAdminHandler, AsyncCallRequestStatusResponseTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, CollectionReloadTest, CollectionTooManyReplicasTest, CreateCollectionCleanupTest, CustomCollectionTest, DeleteReplicaTest, DistribJoinFromCollectionTest, LeaderElectionContextKeyTest, LeaderFailoverAfterPartitionTest, MissingSegmentRecoveryTest, MoveReplicaHDFSTest, MoveReplicaTest, MultiThreadedOCPTest, RemoteQueryErrorTest, ReplicationFactorTest, RestartWhileUpdatingTest, RollingRestartTest, SharedFSAutoReplicaFailoverUtilsTest, SimpleCollectionCreateDeleteTest, SolrXmlInZkTest, TestCloudDeleteByQuery, TestCloudInspectUtil, TestCloudPseudoReturnFields, TestCloudRecovery, TestCollectionsAPIViaSolrCloudCluster, TestConfigSetsAPIExclusivity, TestDistribDocBasedVersion, TestDownShardTolerantSearch, TestHdfsCloudBackupRestore, TestLeaderElectionZkExpiry, TestLockTree, TestMiniSolrCloudClusterSSL, TestPullReplica, TestRandomRequestDistribution, TestReplicaProperties, TestSSLRandomization, TestSizeLimitedDistributedMap, TlogReplayBufferedWhileIndexingTest, VMParamsZkACLAndCredentialsProvidersTest, TestPolicyCloud, HdfsChaosMonkeyNothingIsSafeTest, HdfsChaosMonkeySafeLeaderTest, HdfsCollectionsAPIDistributedZkTest, HdfsRecoverLeaseTest, HdfsSyncSliceTest, HdfsThreadLeakTest, HdfsTlogReplayBufferedWhileIndexingTest, HdfsWriteToMultipleCollectionsTest, ZkStateWriterTest, TestBackupRepositoryFactory, TestConfigOverlay, TestConfigSetImmutable, TestInitParams, TestReloadAndDeleteDocs, TestSolrConfigHandler, SearchHandlerTest, TestBlobHandler, TestHdfsBackupRestoreCore, TestSQLHandlerNonCloud, TestSolrConfigHandlerCloud, DistributedFacetPivotLargeTest, DistributedFacetPivotWhiteBoxTest, DistributedQueryComponentOptimizationTest, TestExpandComponent, TestPivotHelperCode, TestSlowCompositeReaderWrapper, UninvertDocValuesMergePolicyTest, TestLegacyNumericUtils, TestLegacyTerms, JvmMetricsTest, SolrMetricManagerTest, SolrMetricReporterTest, SolrJmxReporterTest, SolrSlf4jReporterTest, TestIntervalFaceting, TestStreamBody, TestMacroExpander, SmileWriterTest, TestCustomDocTransformer, TestExportWriter, TestPushWriter, TestRestManager, TestDynamicFieldResource, TestFieldTypeCollectionResource, TestFieldTypeResource, TestSchemaResource, TestSchemaSimilarityResource, TestUniqueKeyFieldResource, TestManagedStopFilterFactory, TestManagedSynonymGraphFilterFactory, BooleanFieldTest, ChangedSchemaMergeTest, DocValuesMissingTest, PreAnalyzedFieldManagedSchemaCloudTest, TestBulkSchemaConcurrent, TestHalfAndHalfDocValues, TestManagedSchema, TestManagedSchemaThreadSafety, TestPointFields, TestInitQParser, TestMissingGroups, TestNoOpRegenerator, TestPayloadCheckQParserPlugin, TestSimpleQParserPlugin, TestTrieFacet, TestXmlQParserPlugin, TestMinMaxOnMultiValuedField, TestOrdValues, BJQParserTest, TestCloudNestedDocsSort, TestExactSharedStatsCache, TestExactStatsCache, TestLRUStatsCache, TestRuleBasedAuthorizationPlugin, TestImpersonationWithHadoopAuth]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestImpersonationWithHadoopAuth -Dtests.seed=9E12AA94B7B14D2B -Dtests.slow=true -Dtests.locale=es-UY -Dtests.timezone=Atlantic/Cape_Verde -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | TestImpersonationWithHadoopAuth (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth: 
   [junit4]    >    1) Thread[id=36300, name=jetty-launcher-6753-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]    >    2) Thread[id=36296, name=jetty-launcher-6753-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9E12AA94B7B14D2B]:0)
   [junit4] Completed [711/734 (2!)] on J1 in 45.96s, 10 tests, 1 error, 1 skipped <<< FAILURES!

[...truncated 46895 lines...]

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1446 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1446/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC

1 tests failed.
FAILED:  org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testExecutorStream

Error Message:
Error from server at http://127.0.0.1:53067/solr/mainCorpus_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/mainCorpus_shard2_replica_n2/update. Reason: <pre>    Can not find: /solr/mainCorpus_shard2_replica_n2/update</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at http://127.0.0.1:53067/solr/mainCorpus_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/mainCorpus_shard2_replica_n2/update. Reason:
<pre>    Can not find: /solr/mainCorpus_shard2_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([745EC4597C141023:569E45A25F7E3A33]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:539)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:993)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
	at org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testExecutorStream(StreamExpressionTest.java:7472)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:53067/solr/mainCorpus_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/mainCorpus_shard2_replica_n2/update. Reason:
<pre>    Can not find: /solr/mainCorpus_shard2_replica_n2/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:591)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:516)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more




Build Log:
[...truncated 14771 lines...]
   [junit4] Suite: org.apache.solr.client.solrj.io.stream.StreamExpressionTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.io.stream.StreamExpressionTest_745EC4597C141023-001/init-core-data-001
   [junit4]   2> 151373 WARN  (SUITE-StreamExpressionTest-seed#[745EC4597C141023]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=63 numCloses=63
   [junit4]   2> 151373 INFO  (SUITE-StreamExpressionTest-seed#[745EC4597C141023]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 151374 INFO  (SUITE-StreamExpressionTest-seed#[745EC4597C141023]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 151374 INFO  (SUITE-StreamExpressionTest-seed#[745EC4597C141023]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 151375 INFO  (SUITE-StreamExpressionTest-seed#[745EC4597C141023]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.io.stream.StreamExpressionTest_745EC4597C141023-001/tempDir-001
   [junit4]   2> 151375 INFO  (SUITE-StreamExpressionTest-seed#[745EC4597C141023]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 151376 INFO  (Thread-277) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 151376 INFO  (Thread-277) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 151387 ERROR (Thread-277) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 151478 INFO  (SUITE-StreamExpressionTest-seed#[745EC4597C141023]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:56604
   [junit4]   2> 151499 INFO  (jetty-launcher-213-thread-4) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 151499 INFO  (jetty-launcher-213-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 151499 INFO  (jetty-launcher-213-thread-3) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 151500 INFO  (jetty-launcher-213-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4fee0f35{/solr,null,AVAILABLE}
   [junit4]   2> 151505 INFO  (jetty-launcher-213-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6073b38{HTTP/1.1,[http/1.1]}{127.0.0.1:45216}
   [junit4]   2> 151505 INFO  (jetty-launcher-213-thread-4) [    ] o.e.j.s.Server Started @154471ms
   [junit4]   2> 151505 INFO  (jetty-launcher-213-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45216}
   [junit4]   2> 151506 ERROR (jetty-launcher-213-thread-4) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 151506 INFO  (jetty-launcher-213-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 151506 INFO  (jetty-launcher-213-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 151506 INFO  (jetty-launcher-213-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 151506 INFO  (jetty-launcher-213-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-01T02:47:19.635Z
   [junit4]   2> 151507 INFO  (jetty-launcher-213-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7a76a40e{/solr,null,AVAILABLE}
   [junit4]   2> 151507 INFO  (jetty-launcher-213-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@34cccbff{HTTP/1.1,[http/1.1]}{127.0.0.1:34553}
   [junit4]   2> 151507 INFO  (jetty-launcher-213-thread-2) [    ] o.e.j.s.Server Started @154472ms
   [junit4]   2> 151507 INFO  (jetty-launcher-213-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34553}
   [junit4]   2> 151507 ERROR (jetty-launcher-213-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 151507 INFO  (jetty-launcher-213-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 151507 INFO  (jetty-launcher-213-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 151508 INFO  (jetty-launcher-213-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 151508 INFO  (jetty-launcher-213-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-01T02:47:19.637Z
   [junit4]   2> 151511 INFO  (jetty-launcher-213-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@18806c0c{/solr,null,AVAILABLE}
   [junit4]   2> 151511 INFO  (jetty-launcher-213-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5836566e{HTTP/1.1,[http/1.1]}{127.0.0.1:53067}
   [junit4]   2> 151511 INFO  (jetty-launcher-213-thread-3) [    ] o.e.j.s.Server Started @154477ms
   [junit4]   2> 151512 INFO  (jetty-launcher-213-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=53067}
   [junit4]   2> 151512 ERROR (jetty-launcher-213-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 151512 INFO  (jetty-launcher-213-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 151512 INFO  (jetty-launcher-213-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 151512 INFO  (jetty-launcher-213-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 151512 INFO  (jetty-launcher-213-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-01T02:47:19.641Z
   [junit4]   2> 151520 INFO  (jetty-launcher-213-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 151523 INFO  (jetty-launcher-213-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@41438e4a{/solr,null,AVAILABLE}
   [junit4]   2> 151523 INFO  (jetty-launcher-213-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@71be6246{HTTP/1.1,[http/1.1]}{127.0.0.1:42752}
   [junit4]   2> 151523 INFO  (jetty-launcher-213-thread-1) [    ] o.e.j.s.Server Started @154489ms
   [junit4]   2> 151523 INFO  (jetty-launcher-213-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=42752}
   [junit4]   2> 151523 ERROR (jetty-launcher-213-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 151524 INFO  (jetty-launcher-213-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 151524 INFO  (jetty-launcher-213-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 151524 INFO  (jetty-launcher-213-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 151524 INFO  (jetty-launcher-213-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-01T02:47:19.653Z
   [junit4]   2> 151581 INFO  (jetty-launcher-213-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 151588 INFO  (jetty-launcher-213-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 151588 INFO  (jetty-launcher-213-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 151588 INFO  (jetty-launcher-213-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 151603 INFO  (jetty-launcher-213-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56604/solr
   [junit4]   2> 151608 INFO  (jetty-launcher-213-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56604/solr
   [junit4]   2> 151611 INFO  (jetty-launcher-213-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56604/solr
   [junit4]   2> 151612 INFO  (jetty-launcher-213-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56604/solr
   [junit4]   2> 151619 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15ed5d4b0d20005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 151679 WARN  (jetty-launcher-213-thread-4) [n:127.0.0.1:45216_solr    ] o.a.s.c.ZkController The _default configset could not be uploaded. Please provide 'solr.default.confdir' parameter that points to a configset intended to be the default. Current 'solr.default.confdir' value: null
   [junit4]   2> 151679 WARN  (jetty-launcher-213-thread-2) [n:127.0.0.1:34553_solr    ] o.a.s.c.ZkController The _default configset could not be uploaded. Please provide 'solr.default.confdir' parameter that points to a configset intended to be the default. Current 'solr.default.confdir' value: null
   [junit4]   2> 151679 WARN  (jetty-launcher-213-thread-3) [n:127.0.0.1:53067_solr    ] o.a.s.c.ZkController The _default configset could not be uploaded. Please provide 'solr.default.confdir' parameter that points to a configset intended to be the default. Current 'solr.default.confdir' value: null
   [junit4]   2> 151680 WARN  (jetty-launcher-213-thread-1) [n:127.0.0.1:42752_solr    ] o.a.s.c.ZkController The _default configset could not be uploaded. Please provide 'solr.default.confdir' parameter that points to a configset intended to be the default. Current 'solr.default.confdir' value: null
   [junit4]   2> 151693 INFO  (jetty-launcher-213-thread-3) [n:127.0.0.1:53067_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 151693 INFO  (jetty-launcher-213-thread-2) [n:127.0.0.1:34553_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 151694 INFO  (jetty-launcher-213-thread-4) [n:127.0.0.1:45216_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 151696 INFO  (jetty-launcher-213-thread-3) [n:127.0.0.1:53067_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:53067_solr
   [junit4]   2> 151696 INFO  (jetty-launcher-213-thread-1) [n:127.0.0.1:42752_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 151696 INFO  (jetty-launcher-213-thread-2) [n:127.0.0.1:34553_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34553_solr
   [junit4]   2> 151697 INFO  (jetty-launcher-213-thread-4) [n:127.0.0.1:45216_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45216_solr
   [junit4]   2> 151699 INFO  (jetty-launcher-213-thread-3) [n:127.0.0.1:53067_solr    ] o.a.s.c.Overseer Overseer (id=98751351325065226-127.0.0.1:53067_solr-n_0000000000) starting
   [junit4]   2> 151717 INFO  (zkCallback-235-thread-1-processing-n:127.0.0.1:42752_solr) [n:127.0.0.1:42752_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 151728 INFO  (jetty-launcher-213-thread-3) [n:127.0.0.1:53067_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53067_solr
   [junit4]   2> 151747 INFO  (jetty-launcher-213-thread-1) [n:127.0.0.1:42752_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42752_solr
   [junit4]   2> 151829 INFO  (zkCallback-234-thread-1-processing-n:127.0.0.1:45216_solr) [n:127.0.0.1:45216_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 151830 INFO  (zkCallback-232-thread-1-processing-n:127.0.0.1:34553_solr) [n:127.0.0.1:34553_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 151840 INFO  (OverseerStateUpdate-98751351325065226-127.0.0.1:53067_solr-n_0000000000) [n:127.0.0.1:53067_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 151842 INFO  (zkCallback-235-thread-1-processing-n:127.0.0.1:42752_solr) [n:127.0.0.1:42752_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
   [junit4]   2> 151843 INFO  (zkCallback-234-thread-1-processing-n:127.0.0.1:45216_solr) [n:127.0.0.1:45216_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
   [junit4]   2> 151851 INFO  (zkCallback-233-thread-1-processing-n:127.0.0.1:53067_solr) [n:127.0.0.1:53067_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
   [junit4]   2> 151895 INFO  (zkCallback-232-thread-1-processing-n:127.0.0.1:34553_solr) [n:127.0.0.1:34553_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
   [junit4]   2> 151970 INFO  (jetty-launcher-213-thread-2) [n:127.0.0.1:34553_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34553.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cd0f74a
   [junit4]   2> 151977 INFO  (jetty-launcher-213-thread-2) [n:127.0.0.1:34553_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34553.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cd0f74a
   [junit4]   2> 151977 INFO  (jetty-launcher-213-thread-2) [n:127.0.0.1:34553_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34553.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cd0f74a
   [junit4]   2> 151978 INFO  (jetty-launcher-213-thread-2) [n:127.0.0.1:34553_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.io.stream.StreamExpressionTest_745EC4597C141023-001/tempDir-001/node4/.
   [junit4]   2> 152056 INFO  (jetty-launcher-213-thread-3) [n:127.0.0.1:53067_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53067.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cd0f74a
   [junit4]   2> 152084 INFO  (jetty-launcher-213-thread-4) [n:127.0.0.1:45216_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45216.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cd0f74a
   [junit4]   2> 152092 INFO  (jetty-launcher-213-thread-4) [n:127.0.0.1:45216_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45216.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cd0f74a
   [junit4]   2> 152092 INFO  (jetty-launcher-213-thread-4) [n:127.0.0.1:45216_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45216.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cd0f74a
   [junit4]   2> 152093 INFO  (jetty-launcher-213-thread-4) [n:127.0.0.1:45216_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.io.stream.StreamExpressionTest_745EC4597C141023-001/tempDir-001/node3/.
   [junit4]   2> 152104 INFO  (jetty-launcher-213-thread-3) [n:127.0.0.1:53067_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53067.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cd0f74a
   [junit4]   2> 152104 INFO  (jetty-launcher-213-thread-3) [n:127.0.0.1:53067_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53067.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cd0f74a
   [junit4]   2> 152105 INFO  (jetty-launcher-213-thread-3) [n:127.0.0.1:53067_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.io.stream.StreamExpressionTest_745EC4597C141023-001/tempDir-001/node2/.
   [junit4]   2> 152196 INFO  (jetty-launcher-213-thread-1) [n:127.0.0.1:42752_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42752.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cd0f74a
   [junit4]   2> 152206 INFO  (jetty-launcher-213-thread-1) [n:127.0.0.1:42752_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42752.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cd0f74a
   [junit4]   2> 152207 INFO  (jetty-launcher-213-thread-1) [n:127.0.0.1:42752_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42752.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cd0f74a
   [junit4]   2> 152207 INFO  (jetty-launcher-213-thread-1) [n:127.0.0.1:42752_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.io.stream.StreamExpressionTest_745EC4597C141023-001/tempDir-001/node1/.
   [junit4]   2> 152253 INFO  (SUITE-StreamExpressionTest-seed#[745EC4597C141023]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 152255 INFO  (SUITE-StreamExpressionTest-seed#[745EC4597C141023]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56604/solr ready
   [junit4]   2> 152270 INFO  (qtp841901197-1655) [n:127.0.0.1:53067_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 152273 INFO  (OverseerThreadFactory-869-thread-1-processing-n:127.0.0.1:53067_solr) [n:127.0.0.1:53067_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 152387 INFO  (OverseerStateUpdate-98751351325065226-127.0.0.1:53067_solr-n_0000000000) [n:127.0.0.1:53067_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"collection1_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:45216/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 152390 INFO  (OverseerStateUpdate-98751351325065226-127.0.0.1:53067_solr-n_0000000000) [n:127.0.0.1:53067_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"collection1_shard2_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:53067/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 152607 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=collection1_shard1_replica_n1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 152607 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=collection1_shard2_replica_n2&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 152608 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 152608 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 152724 INFO  (zkCallback-234-thread-1-processing-n:127.0.0.1:45216_solr) [n:127.0.0.1:45216_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 152724 INFO  (zkCallback-233-thread-2-processing-n:127.0.0.1:53067_solr) [n:127.0.0.1:53067_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 152724 INFO  (zkCallback-233-thread-1-processing-n:127.0.0.1:53067_solr) [n:127.0.0.1:53067_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 152725 INFO  (zkCallback-234-thread-2-processing-n:127.0.0.1:45216_solr) [n:127.0.0.1:45216_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 153661 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 153666 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 153678 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema [collection1_shard1_replica_n1] Schema name=test
   [junit4]   2> 153678 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.IndexSchema [collection1_shard2_replica_n2] Schema name=test
   [junit4]   2> 153789 WARN  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.IndexSchema Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 153789 WARN  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.IndexSchema Field text is not multivalued and destination for multiple copyFields (3)
   [junit4]   2> 153790 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 153790 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n2' using configuration from collection collection1, trusted=true
   [junit4]   2> 153790 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53067.solr.core.collection1.shard2.replica_n2' (registry 'solr.core.collection1.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cd0f74a
   [junit4]   2> 153790 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 153790 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SolrCore [[collection1_shard2_replica_n2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.io.stream.StreamExpressionTest_745EC4597C141023-001/tempDir-001/node2/collection1_shard2_replica_n2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.io.stream.StreamExpressionTest_745EC4597C141023-001/tempDir-001/node2/./collection1_shard2_replica_n2/data/]
   [junit4]   2> 153799 WARN  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 153799 WARN  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema Field text is not multivalued and destination for multiple copyFields (3)
   [junit4]   2> 153799 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 153799 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n1' using configuration from collection collection1, trusted=true
   [junit4]   2> 153800 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45216.solr.core.collection1.shard1.replica_n1' (registry 'solr.core.collection1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cd0f74a
   [junit4]   2> 153800 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 153800 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [[collection1_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.io.stream.StreamExpressionTest_745EC4597C141023-001/tempDir-001/node3/collection1_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.io.stream.StreamExpressionTest_745EC4597C141023-001/tempDir-001/node3/./collection1_shard1_replica_n1/data/]
   [junit4]   2> 153867 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 153868 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 153870 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 153870 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 153878 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@7590a002[collection1_shard2_replica_n2] main]
   [junit4]   2> 153883 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 153886 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 153888 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 153888 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 153888 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 153889 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580021623835066368
   [junit4]   2> 153891 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 153891 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 153897 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@23122ea4[collection1_shard1_replica_n1] main]
   [junit4]   2> 153899 INFO  (searcherExecutor-877-thread-1-processing-n:127.0.0.1:53067_solr x:collection1_shard2_replica_n2 s:shard2 c:collection1 r:core_node4) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SolrCore [collection1_shard2_replica_n2] Registered new searcher Searcher@7590a002[collection1_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 153900 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 153900 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 153901 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 153902 INFO  (searcherExecutor-878-thread-1-processing-n:127.0.0.1:45216_solr x:collection1_shard1_replica_n1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [collection1_shard1_replica_n1] Registered new searcher Searcher@23122ea4[collection1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 153903 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580021623849746432
   [junit4]   2> 153929 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 153929 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 153929 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45216/solr/collection1_shard1_replica_n1/
   [junit4]   2> 153929 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 153929 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:45216/solr/collection1_shard1_replica_n1/ has no replicas
   [junit4]   2> 153929 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 153942 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45216/solr/collection1_shard1_replica_n1/ shard1
   [junit4]   2> 153947 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 153947 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 153947 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53067/solr/collection1_shard2_replica_n2/
   [junit4]   2> 153947 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 153948 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SyncStrategy http://127.0.0.1:53067/solr/collection1_shard2_replica_n2/ has no replicas
   [junit4]   2> 153948 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 153967 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53067/solr/collection1_shard2_replica_n2/ shard2
   [junit4]   2> 154072 INFO  (zkCallback-234-thread-2-processing-n:127.0.0.1:45216_solr) [n:127.0.0.1:45216_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 154072 INFO  (zkCallback-234-thread-1-processing-n:127.0.0.1:45216_solr) [n:127.0.0.1:45216_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 154072 INFO  (zkCallback-233-thread-1-processing-n:127.0.0.1:53067_solr) [n:127.0.0.1:53067_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 154072 INFO  (zkCallback-233-thread-2-processing-n:127.0.0.1:53067_solr) [n:127.0.0.1:53067_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 154098 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 154120 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 154123 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=collection1_shard1_replica_n1&action=CREATE&numShards=2&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1515
   [junit4]   2> 154123 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=collection1_shard2_replica_n2&action=CREATE&numShards=2&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1515
   [junit4]   2> 154139 INFO  (qtp841901197-1655) [n:127.0.0.1:53067_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 154237 INFO  (zkCallback-233-thread-1-processing-n:127.0.0.1:53067_solr) [n:127.0.0.1:53067_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 154237 INFO  (zkCallback-234-thread-2-processing-n:127.0.0.1:45216_solr) [n:127.0.0.1:45216_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 154237 INFO  (zkCallback-233-thread-2-processing-n:127.0.0.1:53067_solr) [n:127.0.0.1:53067_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 154237 INFO  (zkCallback-234-thread-1-processing-n:127.0.0.1:45216_solr) [n:127.0.0.1:45216_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 154293 INFO  (OverseerCollectionConfigSetProcessor-98751351325065226-127.0.0.1:53067_solr-n_0000000000) [n:127.0.0.1:53067_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 155141 INFO  (qtp841901197-1655) [n:127.0.0.1:53067_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=2871
   [junit4]   2> 155141 INFO  (SUITE-StreamExpressionTest-seed#[745EC4597C141023]-worker) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):90
   [junit4]   2> 155142 INFO  (SUITE-StreamExpressionTest-seed#[745EC4597C141023]-worker) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
   [junit4]   2> 155174 INFO  (TEST-StreamExpressionTest.testParallelDaemonCommitStream-seed#[745EC4597C141023]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testParallelDaemonCommitStream
   [junit4]   2> 155181 INFO  (qtp841901197-1653) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_n2]  webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:45216/solr/collection1_shard1_replica_n1/&wt=javabin&version=2}{deleteByQuery=*:* (-1580021625188777984)} 0 1
   [junit4]   2> 155183 INFO  (qtp841901197-1654) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1580021625191923712,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 155183 INFO  (qtp841901197-1654) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 155183 INFO  (qtp841901197-1654) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 155183 INFO  (qtp841901197-1654) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:45216/solr/collection1_shard1_replica_n1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
   [junit4]   2> 155183 INFO  (qtp2027366912-1639) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1580021625191923712,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 155183 INFO  (qtp2027366912-1639) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 155184 INFO  (qtp2027366912-1639) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 155184 INFO  (qtp2027366912-1639) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:45216/solr/collection1_shard1_replica_n1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
   [junit4]   2> 155184 INFO  (qtp2027366912-1635) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=collection1:4&commit=true&wt=javabin&version=2}{deleteByQuery=*:* (-1580021625184583680),commit=} 0 8
   [junit4]   2> 155185 INFO  (qtp841901197-1652) [n:127.0.0.1:53067_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=parallelDestinationCollection1&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 155188 INFO  (OverseerThreadFactory-869-thread-2-processing-n:127.0.0.1:53067_solr) [n:127.0.0.1:53067_solr    ] o.a.s.c.CreateCollectionCmd Create collection parallelDestinationCollection1
   [junit4]   2> 155404 INFO  (OverseerStateUpdate-98751351325065226-127.0.0.1:53067_solr-n_0000000000) [n:127.0.0.1:53067_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"parallelDestinationCollection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"parallelDestinationCollection1_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:34553/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 155411 INFO  (OverseerStateUpdate-98751351325065226-127.0.0.1:53067_solr-n_0000000000) [n:127.0.0.1:53067_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"parallelDestinationCollection1",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"parallelDestinationCollection1_shard2_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:45216/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 155615 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=parallelDestinationCollection1_shard2_replica_n2&action=CREATE&numShards=2&collection=parallelDestinationCollection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 155618 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=parallelDestinationCollection1_shard1_replica_n1&action=CREATE&numShards=2&collection=parallelDestinationCollection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 155619 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 155738 INFO  (zkCallback-234-thread-2-processing-n:127.0.0.1:45216_solr) [n:127.0.0.1:45216_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/parallelDestinationCollection1/state.json] for collection [parallelDestinationCollection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 155738 INFO  (zkCallback-234-thread-1-processing-n:127.0.0.1:45216_solr) [n:127.0.0.1:45216_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/parallelDestinationCollection1/state.json] for collection [parallelDestinationCollection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 155740 INFO  (zkCallback-232-thread-1-processing-n:127.0.0.1:34553_solr) [n:127.0.0.1:34553_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/parallelDestinationCollection1/state.json] for collection [parallelDestinationCollection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 155741 INFO  (zkCallback-232-thread-2-processing-n:127.0.0.1:34553_solr) [n:127.0.0.1:34553_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/parallelDestinationCollection1/state.json] for collection [parallelDestinationCollection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 156635 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 156635 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 156661 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.s.IndexSchema [parallelDestinationCollection1_shard1_replica_n1] Schema name=test
   [junit4]   2> 156661 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.s.IndexSchema [parallelDestinationCollection1_shard2_replica_n2] Schema name=test
   [junit4]   2> 156738 WARN  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.s.IndexSchema Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 156738 WARN  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.s.IndexSchema Field text is not multivalued and destination for multiple copyFields (3)
   [junit4]   2> 156739 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 156739 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'parallelDestinationCollection1_shard1_replica_n1' using configuration from collection parallelDestinationCollection1, trusted=true
   [junit4]   2> 156739 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34553.solr.core.parallelDestinationCollection1.shard1.replica_n1' (registry 'solr.core.parallelDestinationCollection1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cd0f74a
   [junit4]   2> 156739 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 156739 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.c.SolrCore [[parallelDestinationCollection1_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.io.stream.StreamExpressionTest_745EC4597C141023-001/tempDir-001/node4/parallelDestinationCollection1_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.io.stream.StreamExpressionTest_745EC4597C141023-001/tempDir-001/node4/./parallelDestinationCollection1_shard1_replica_n1/data/]
   [junit4]   2> 156740 WARN  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.s.IndexSchema Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 156740 WARN  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.s.IndexSchema Field text is not multivalued and destination for multiple copyFields (3)
   [junit4]   2> 156741 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 156741 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'parallelDestinationCollection1_shard2_replica_n2' using configuration from collection parallelDestinationCollection1, trusted=true
   [junit4]   2> 156742 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45216.solr.core.parallelDestinationCollection1.shard2.replica_n2' (registry 'solr.core.parallelDestinationCollection1.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5cd0f74a
   [junit4]   2> 156742 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 156742 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.c.SolrCore [[parallelDestinationCollection1_shard2_replica_n2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.io.stream.StreamExpressionTest_745EC4597C141023-001/tempDir-001/node3/parallelDestinationCollection1_shard2_replica_n2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.io.stream.StreamExpressionTest_745EC4597C141023-001/tempDir-001/node3/./parallelDestinationCollection1_shard2_replica_n2/data/]
   [junit4]   2> 156801 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 156801 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 156804 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 156804 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 156805 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@55344d6c[parallelDestinationCollection1_shard1_replica_n1] main]
   [junit4]   2> 156807 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 156807 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 156808 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 156808 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580021626895859712
   [junit4]   2> 156827 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 156827 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 156827 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:34553/solr/parallelDestinationCollection1_shard1_replica_n1/
   [junit4]   2> 156827 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 156827 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:34553/solr/parallelDestinationCollection1_shard1_replica_n1/ has no replicas
   [junit4]   2> 156827 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 156833 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:34553/solr/parallelDestinationCollection1_shard1_replica_n1/ shard1
   [junit4]   2> 156849 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 156849 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 156850 INFO  (searcherExecutor-887-thread-1-processing-n:127.0.0.1:34553_solr x:parallelDestinationCollection1_shard1_replica_n1 s:shard1 c:parallelDestinationCollection1 r:core_node3) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.c.SolrCore [parallelDestinationCollection1_shard1_replica_n1] Registered new searcher Searcher@55344d6c[parallelDestinationCollection1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 156851 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 156852 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 156853 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@1452ac71[parallelDestinationCollection1_shard2_replica_n2] main]
   [junit4]   2> 156855 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 156855 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 156856 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 156856 INFO  (searcherExecutor-888-thread-1-processing-n:127.0.0.1:45216_solr x:parallelDestinationCollection1_shard2_replica_n2 s:shard2 c:parallelDestinationCollection1 r:core_node4) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.c.SolrCore [parallelDestinationCollection1_shard2_replica_n2] Registered new searcher Searcher@1452ac71[parallelDestinationCollection1_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 156857 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580021626947239936
   [junit4]   2> 156875 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 156876 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 156876 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45216/solr/parallelDestinationCollection1_shard2_replica_n2/
   [junit4]   2> 156876 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 156876 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.c.SyncStrategy http://127.0.0.1:45216/solr/parallelDestinationCollection1_shard2_replica_n2/ has no replicas
   [junit4]   2> 156876 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 156902 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45216/solr/parallelDestinationCollection1_shard2_replica_n2/ shard2
   [junit4]   2> 157011 INFO  (zkCallback-232-thread-2-processing-n:127.0.0.1:34553_solr) [n:127.0.0.1:34553_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/parallelDestinationCollection1/state.json] for collection [parallelDestinationCollection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 157011 INFO  (zkCallback-232-thread-1-processing-n:127.0.0.1:34553_solr) [n:127.0.0.1:34553_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/parallelDestinationCollection1/state.json] for collection [parallelDestinationCollection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 157012 INFO  (zkCallback-234-thread-1-processing-n:127.0.0.1:45216_solr) [n:127.0.0.1:45216_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/parallelDestinationCollection1/state.json] for collection [parallelDestinationCollection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 157012 INFO  (zkCallback-234-thread-2-processing-n:127.0.0.1:45216_solr) [n:127.0.0.1:45216_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/parallelDestinationCollection1/state.json] for collection [parallelDestinationCollection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 157035 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 157037 INFO  (qtp1077087270-1641) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=parallelDestinationCollection1_shard1_replica_n1&action=CREATE&numShards=2&collection=parallelDestinationCollection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1418
   [junit4]   2> 157054 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 157057 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=parallelDestinationCollection1_shard2_replica_n2&action=CREATE&numShards=2&collection=parallelDestinationCollection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1442
   [junit4]   2> 157060 INFO  (qtp841901197-1652) [n:127.0.0.1:53067_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 157160 INFO  (zkCallback-234-thread-2-processing-n:127.0.0.1:45216_solr) [n:127.0.0.1:45216_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/parallelDestinationCollection1/state.json] for collection [parallelDestinationCollection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 157160 INFO  (zkCallback-234-thread-1-processing-n:127.0.0.1:45216_solr) [n:127.0.0.1:45216_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/parallelDestinationCollection1/state.json] for collection [parallelDestinationCollection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 157160 INFO  (zkCallback-232-thread-1-processing-n:127.0.0.1:34553_solr) [n:127.0.0.1:34553_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/parallelDestinationCollection1/state.json] for collection [parallelDestinationCollection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 157161 INFO  (zkCallback-232-thread-2-processing-n:127.0.0.1:34553_solr) [n:127.0.0.1:34553_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/parallelDestinationCollection1/state.json] for collection [parallelDestinationCollection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 157189 INFO  (OverseerCollectionConfigSetProcessor-98751351325065226-127.0.0.1:53067_solr-n_0000000000) [n:127.0.0.1:53067_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 158071 INFO  (qtp841901197-1652) [n:127.0.0.1:53067_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=parallelDestinationCollection1&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=2886
   [junit4]   2> 158071 INFO  (TEST-StreamExpressionTest.testParallelDaemonCommitStream-seed#[745EC4597C141023]) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: parallelDestinationCollection1 failOnTimeout:true timeout (sec):90
   [junit4]   2> 158072 INFO  (TEST-StreamExpressionTest.testParallelDaemonCommitStream-seed#[745EC4597C141023]) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: parallelDestinationCollection1
   [junit4]   2> 158078 INFO  (qtp2027366912-1637) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=collection1:4&wt=javabin&version=2}{add=[0 (1580021628223356928), 4 (1580021628226502656), 1 (1580021628226502657)]} 0 3
   [junit4]   2> 158078 INFO  (qtp841901197-1661) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_n2]  webapp=/solr path=/update params={_stateVer_=collection1:4&wt=javabin&version=2}{add=[2 (1580021628223356928), 3 (1580021628226502656)]} 0 3
   [junit4]   2> 158083 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1580021628232794112,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 158083 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@60dcd19e commitCommandVersion:1580021628232794112
   [junit4]   2> 158086 INFO  (qtp841901197-1655) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1580021628235939840,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 158086 INFO  (qtp841901197-1655) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@4bd1bfe7 commitCommandVersion:1580021628235939840
   [junit4]   2> 158121 INFO  (qtp841901197-1655) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@4f9b0f2c[collection1_shard2_replica_n2] main]
   [junit4]   2> 158121 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4a7d37b5[collection1_shard1_replica_n1] main]
   [junit4]   2> 158121 INFO  (qtp841901197-1655) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 158121 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 158123 INFO  (searcherExecutor-878-thread-1-processing-n:127.0.0.1:45216_solr x:collection1_shard1_replica_n1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.SolrCore [collection1_shard1_replica_n1] Registered new searcher Searcher@4a7d37b5[collection1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C3)))}
   [junit4]   2> 158123 INFO  (searcherExecutor-877-thread-1-processing-n:127.0.0.1:53067_solr x:collection1_shard2_replica_n2 s:shard2 c:collection1 r:core_node4) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.SolrCore [collection1_shard2_replica_n2] Registered new searcher Searcher@4f9b0f2c[collection1_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C2)))}
   [junit4]   2> 158123 INFO  (qtp2027366912-1634) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_n1]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53067/solr/collection1_shard2_replica_n2/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 39
   [junit4]   2> 158123 INFO  (qtp841901197-1655) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_n2]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53067/solr/collection1_shard2_replica_n2/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 36
   [junit4]   2> 158123 INFO  (qtp841901197-1662) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_n2]  webapp=/solr path=/update params={_stateVer_=collection1:4&commit=true&wt=javabin&version=2}{commit=} 0 44
   [junit4]   2> 158151 INFO  (TEST-StreamExpressionTest.testParallelDaemonCommitStream-seed#[745EC4597C141023]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 158152 INFO  (TEST-StreamExpressionTest.testParallelDaemonCommitStream-seed#[745EC4597C141023]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56604/solr ready
   [junit4]   2> 158162 INFO  (qtp841901197-1654) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.S.Request [collection1_shard2_replica_n2]  webapp=/solr path=/stream params={distrib=false&workerID=0&indent=off&numWorkers=2&expr=daemon(commit(parallelDestinationCollection1,zkHost%3D"127.0.0.1:56604/solr",batchSize%3D0,waitFlush%3Dfalse,waitSearcher%3Dfalse,softCommit%3Dfalse,update(parallelDestinationCollection1,zkHost%3D"127.0.0.1:56604/solr",batchSize%3D2,search(collection1,q%3D"*:*",fl%3D"id,a_s,a_i,a_f,s_multi,i_multi",sort%3D"a_f+asc,+a_i+asc",partitionKeys%3Da_f,zkHost%3D"127.0.0.1:56604/solr"))),id%3Dtest,runInterval%3D1000,queueSize%3D0,terminate%3Dfalse)&wt=json&version=2.2} status=0 QTime=3
   [junit4]   2> 158163 INFO  (qtp2027366912-1636) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.S.Request [collection1_shard1_replica_n1]  webapp=/solr path=/stream params={distrib=false&workerID=1&indent=off&numWorkers=2&expr=daemon(commit(parallelDestinationCollection1,zkHost%3D"127.0.0.1:56604/solr",batchSize%3D0,waitFlush%3Dfalse,waitSearcher%3Dfalse,softCommit%3Dfalse,update(parallelDestinationCollection1,zkHost%3D"127.0.0.1:56604/solr",batchSize%3D2,search(collection1,q%3D"*:*",fl%3D"id,a_s,a_i,a_f,s_multi,i_multi",sort%3D"a_f+asc,+a_i+asc",partitionKeys%3Da_f,zkHost%3D"127.0.0.1:56604/solr"))),id%3Dtest,runInterval%3D1000,queueSize%3D0,terminate%3Dfalse)&wt=json&version=2.2} status=0 QTime=3
   [junit4]   2> 158189 INFO  (Thread-318) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 158190 INFO  (Thread-318) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56604/solr ready
   [junit4]   2> 158219 INFO  (qtp2027366912-1638) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.S.Request [collection1_shard1_replica_n1]  webapp=/solr path=/select params={q=*:*&distrib=false&fl=id,a_s,a_i,a_f,s_multi,i_multi&sort=a_f+asc,+a_i+asc&partitionKeys=a_f&fq={!hash+workers%3D2+worker%3D1}&wt=json&version=2.2} hits=0 status=0 QTime=10
   [junit4]   2> 158220 INFO  (qtp2027366912-1638) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.c.S.Request [collection1_shard1_replica_n1]  webapp=/solr path=/select params={q=*:*&distrib=false&fl=id,a_s,a_i,a_f,s_multi,i_multi&sort=a_f+asc,+a_i+asc&partitionKeys=a_f&fq={!hash+workers%3D2+worker%3D0}&wt=json&version=2.2} hits=3 status=0 QTime=0
   [junit4]   2> 158233 INFO  (qtp841901197-1652) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.S.Request [collection1_shard2_replica_n2]  webapp=/solr path=/select params={q=*:*&distrib=false&fl=id,a_s,a_i,a_f,s_multi,i_multi&sort=a_f+asc,+a_i+asc&partitionKeys=a_f&fq={!hash+workers%3D2+worker%3D1}&wt=json&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 158238 INFO  (qtp841901197-1655) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.S.Request [collection1_shard2_replica_n2]  webapp=/solr path=/select params={q=*:*&distrib=false&fl=id,a_s,a_i,a_f,s_multi,i_multi&sort=a_f+asc,+a_i+asc&partitionKeys=a_f&fq={!hash+workers%3D2+worker%3D0}&wt=json&version=2.2} hits=2 status=0 QTime=5
   [junit4]   2> 158238 INFO  (qtp841901197-1659) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.S.Request [collection1_shard2_replica_n2]  webapp=/solr path=/stream params={indent=off&action=list&wt=json&version=2.2} status=0 QTime=6
   [junit4]   2> 158263 INFO  (qtp1077087270-1656) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [parallelDestinationCollection1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[0 (1580021628419440640)]} 0 2
   [junit4]   2> 158266 INFO  (qtp2027366912-1638) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [parallelDestinationCollection1_shard2_replica_n2]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[2 (1580021628403712000)]} 0 20
   [junit4]   2> 158272 INFO  (qtp1077087270-1643) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [parallelDestinationCollection1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[1 (1580021628429926400)]} 0 0
   [junit4]   2> 158275 INFO  (qtp2027366912-1633) [n:127.0.0.1:45216_solr c:parallelDestinationCollection1 s:shard2 r:core_node4 x:parallelDestinationCollection1_shard2_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [parallelDestinationCollection1_shard2_replica_n2]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[3 (1580021628432023552)]} 0 1
   [junit4]   2> 158276 INFO  (qtp1077087270-1643) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [parallelDestinationCollection1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[4 (1580021628435169280)]} 0 0
   [junit4]   2> 158291 INFO  (Thread-317) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 158297 INFO  (Thread-317) [n:127.0.0.1:53067_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n2] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56604/solr ready
   [junit4]   2> 158304 INFO  (qtp1077087270-1646) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 r:core_node3 x:parallelDestinationCollection1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1580021628464529408,optimize=false,openSearcher=true,waitSearcher=false,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 158304 INFO  (qtp1077087270-1646) [n:127.0.0.1:34553_solr c:parallelDestinationCollection1 s:shard1 

[...truncated too long message...]

stry=solr.core.mainCorpus.shard1.replica_n1, tag=1920351821
   [junit4]   2> 259237 INFO  (coreCloseExecutor-1070-thread-3) [n:127.0.0.1:45216_solr c:mainCorpus s:shard1 r:core_node2 x:mainCorpus_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1b7d99ad: rootName = solr_45216, domain = solr.core.mainCorpus.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.mainCorpus.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@7ef6fa96
   [junit4]   2> 259244 INFO  (coreCloseExecutor-1070-thread-3) [n:127.0.0.1:45216_solr c:mainCorpus s:shard1 r:core_node2 x:mainCorpus_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.mainCorpus.shard1.leader, tag=1920351821
   [junit4]   2> 259244 INFO  (coreCloseExecutor-1070-thread-1) [n:127.0.0.1:45216_solr c:collection1 s:shard1 r:core_node3 x:collection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard1.leader, tag=859936539
   [junit4]   2> 259244 INFO  (coreCloseExecutor-1070-thread-3) [n:127.0.0.1:45216_solr c:mainCorpus s:shard1 r:core_node2 x:mainCorpus_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter close.
   [junit4]   2> 259244 INFO  (coreCloseExecutor-1070-thread-3) [n:127.0.0.1:45216_solr c:mainCorpus s:shard1 r:core_node2 x:mainCorpus_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@740e91e6 commitCommandVersion:0
   [junit4]   2> 260508 WARN  (zkCallback-232-thread-2-processing-n:127.0.0.1:34553_solr) [n:127.0.0.1:34553_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 260509 INFO  (jetty-closer-214-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7a76a40e{/solr,null,UNAVAILABLE}
   [junit4]   2> 260739 WARN  (zkCallback-233-thread-1-processing-n:127.0.0.1:53067_solr) [n:127.0.0.1:53067_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 260740 INFO  (jetty-closer-214-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@18806c0c{/solr,null,UNAVAILABLE}
   [junit4]   2> 260762 WARN  (zkCallback-234-thread-1-processing-n:127.0.0.1:45216_solr) [n:127.0.0.1:45216_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 260763 INFO  (jetty-closer-214-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4fee0f35{/solr,null,UNAVAILABLE}
   [junit4]   2> 260764 ERROR (SUITE-StreamExpressionTest-seed#[745EC4597C141023]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 260765 INFO  (SUITE-StreamExpressionTest-seed#[745EC4597C141023]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:56604 56604
   [junit4]   2> 270812 INFO  (Thread-277) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:56604 56604
   [junit4]   2> 270815 WARN  (Thread-277) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	97	/solr/aliases.json
   [junit4]   2> 	97	/solr/clusterprops.json
   [junit4]   2> 	12	/solr/configs/conf
   [junit4]   2> 	8	/solr/configs/ml
   [junit4]   2> 	4	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	97	/solr/clusterstate.json
   [junit4]   2> 	40	/solr/collections/destinationCollection/state.json
   [junit4]   2> 	26	/solr/collections/parallelDestinationCollection1/state.json
   [junit4]   2> 	24	/solr/collections/destination/state.json
   [junit4]   2> 	22	/solr/collections/mainCorpus/state.json
   [junit4]   2> 	20	/solr/collections/parallelDestinationCollection/state.json
   [junit4]   2> 	20	/solr/collections/workQueue/state.json
   [junit4]   2> 	12	/solr/collections/collection1/state.json
   [junit4]   2> 	10	/solr/collections/uknownCollection/state.json
   [junit4]   2> 	10	/solr/collections/modelCollection/state.json
   [junit4]   2> 	10	/solr/collections/checkpointCollection/state.json
   [junit4]   2> 	2	/solr/overseer_elect/election/98751351325065226-127.0.0.1:53067_solr-n_0000000000
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	97	/solr/live_nodes
   [junit4]   2> 	97	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J0/temp/solr.client.solrj.io.stream.StreamExpressionTest_745EC4597C141023-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {name_s=PostingsFormat(name=Direct), expr_s=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), terms_ss=PostingsFormat(name=Asserting), field_s=PostingsFormat(name=MockRandom), join2_s=PostingsFormat(name=Asserting), multiDefault=PostingsFormat(name=Asserting), subject=PostingsFormat(name=Asserting), a1_s=PostingsFormat(name=Asserting), a2_s=PostingsFormat(name=Direct), level1_s=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), body_t=PostingsFormat(name=Direct), s_multi=PostingsFormat(name=Direct), col_s=PostingsFormat(name=Asserting), a_ss=PostingsFormat(name=MockRandom), level2_s=PostingsFormat(name=MockRandom), a_s=PostingsFormat(name=Direct), tv_text=PostingsFormat(name=MockRandom), term_s=PostingsFormat(name=Asserting), a_t=PostingsFormat(name=MockRandom), text_s=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), text=PostingsFormat(name=Direct), featureSet_s=PostingsFormat(name=Asserting), checkpoint_ss=PostingsFormat(name=Direct), test_t=PostingsFormat(name=Direct), ident_s=PostingsFormat(name=Asserting), whitetok=PostingsFormat(name=Direct), side_s=PostingsFormat(name=Direct)}, docValues:{name_s=DocValuesFormat(name=Lucene70), expr_s=DocValuesFormat(name=Direct), iteration_i=DocValuesFormat(name=Direct), terms_ss=DocValuesFormat(name=Memory), join2_s=DocValuesFormat(name=Memory), multiDefault=DocValuesFormat(name=Memory), a_f=DocValuesFormat(name=Memory), intDefault=DocValuesFormat(name=Lucene70), falseNegative_i=DocValuesFormat(name=Lucene70), a2_s=DocValuesFormat(name=Lucene70), a_i=DocValuesFormat(name=Direct), level1_s=DocValuesFormat(name=Direct), a_s=DocValuesFormat(name=Lucene70), id=DocValuesFormat(name=Direct), alpha_d=DocValuesFormat(name=Lucene70), i_multi=DocValuesFormat(name=Direct), idf_d=DocValuesFormat(name=Memory), b_ls=DocValuesFormat(name=Memory), checkpoint_ss=DocValuesFormat(name=Lucene70), ident_s=DocValuesFormat(name=Memory), order_i=DocValuesFormat(name=Memory), error_d=DocValuesFormat(name=Direct), truePositive_i=DocValuesFormat(name=Direct), side_s=DocValuesFormat(name=Lucene70), miles_i=DocValuesFormat(name=Memory), field_s=DocValuesFormat(name=Lucene70), price_f=DocValuesFormat(name=Memory), a1_s=DocValuesFormat(name=Memory), join1_i=DocValuesFormat(name=Direct), test_i=DocValuesFormat(name=Memory), join3_i=DocValuesFormat(name=Lucene70), falsePositive_i=DocValuesFormat(name=Lucene70), field_i=DocValuesFormat(name=Memory), s_multi=DocValuesFormat(name=Lucene70), col_s=DocValuesFormat(name=Memory), a_ss=DocValuesFormat(name=Lucene70), level2_s=DocValuesFormat(name=Lucene70), score_f=DocValuesFormat(name=Lucene70), term_s=DocValuesFormat(name=Memory), text_s=DocValuesFormat(name=Direct), timestamp=DocValuesFormat(name=Lucene70), idfs_ds=DocValuesFormat(name=Lucene70), weights_ds=DocValuesFormat(name=Lucene70), featureSet_s=DocValuesFormat(name=Memory), trueNegative_i=DocValuesFormat(name=Lucene70), index_i=DocValuesFormat(name=Memory), test_dt=DocValuesFormat(name=Direct), out_i=DocValuesFormat(name=Memory), _version_=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=1805, maxMBSortInHeap=5.274671639834752, sim=RandomSimilarity(queryNorm=false): {}, locale=mt-MT, timezone=Pacific/Easter
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=381050296,total=529530880
   [junit4]   2> NOTE: All tests run in this JVM: [JettyWebappTest, TestValidatingJsonMap, AndEvaluatorTest, CoalesceEvaluatorTest, OperationsTest, TestSpellCheckResponse, SolrZkClientTest, TestEmbeddedSolrServer, TestV2Request, HyperbolicCosineEvaluatorTest, HttpSolrClientConPoolTest, TestClusteringResponse, TestConfigSetAdminRequest, EqualToEvaluatorTest, AbsoluteValueEvaluatorTest, TestSolrProperties, NamedListTest, TestJavaBinCodec, UniformDistributionEvaluatorTest, OrEvaluatorTest, CloudSolrClientCacheTest, TestSolrJErrorHandling, CorrelationEvaluatorTest, TestDocumentObjectBinder, DivideEvaluatorTest, TestPathTrie, SolrExampleEmbeddedTest, JdbcDriverTest, TestHash, SolrExampleStreamingTest, TestUpdateRequestCodec, SquareRootEvaluatorTest, ConcurrentUpdateSolrClientTest, GreaterThanEqualToEvaluatorTest, RegressionEvaluatorTest, SolrExampleBinaryTest, SolrExampleXMLTest, SolrExampleJettyTest, LargeVolumeBinaryJettyTest, TestBatchUpdate, LargeVolumeEmbeddedTest, LargeVolumeJettyTest, TermsResponseTest, MergeIndexesEmbeddedTest, QueryResponseTest, TestCoreAdmin, SolrQueryTest, TestFastInputStream, SolrDocumentTest, FieldAnalysisResponseTest, AnlysisResponseBaseTest, SolrSchemalessExampleTest, CloudSolrClientTest, StreamExpressionTest]
   [junit4] Completed [145/145 (1!)] on J0 in 119.48s, 117 tests, 1 error <<< FAILURES!

[...truncated 45894 lines...]