You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/11/12 13:49:21 UTC

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1530 - Still Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1530/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseG1GC

2 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:     1) Thread[id=16042, name=jetty-launcher-3193-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   1) Thread[id=16042, name=jetty-launcher-3193-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
	at __randomizedtesting.SeedInfo.seed([70AC4789D20462D7]:0)


FAILED:  org.apache.solr.client.solrj.impl.CloudSolrClientTest.testRetryUpdatesWhenClusterStateIsStale

Error Message:
Error from server at http://127.0.0.1:35985/solr/stale_state_test_col_shard1_replica_n1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 </title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /solr/stale_state_test_col_shard1_replica_n1/update. Reason: <pre>    Can not find: /solr/stale_state_test_col_shard1_replica_n1/update</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:35985/solr/stale_state_test_col_shard1_replica_n1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/stale_state_test_col_shard1_replica_n1/update. Reason:
<pre>    Can not find: /solr/stale_state_test_col_shard1_replica_n1/update</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([7A6529AF03C59754:CE54B147E02CE178]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:559)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1016)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
	at org.apache.solr.client.solrj.impl.CloudSolrClientTest.testRetryUpdatesWhenClusterStateIsStale(CloudSolrClientTest.java:844)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.junit.rules.ExpectedException$ExpectedExceptionStatement.evaluate(ExpectedException.java:110)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12344 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
   [junit4]   2> 1663888 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[70AC4789D20462D7]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_70AC4789D20462D7-001/init-core-data-001
   [junit4]   2> 1663889 WARN  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[70AC4789D20462D7]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
   [junit4]   2> 1663889 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[70AC4789D20462D7]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1663892 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[70AC4789D20462D7]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1663934 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[70AC4789D20462D7]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_70AC4789D20462D7-001/tempDir-001
   [junit4]   2> 1663935 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[70AC4789D20462D7]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1663935 INFO  (Thread-5274) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1663935 INFO  (Thread-5274) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1663938 ERROR (Thread-5274) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1664039 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[70AC4789D20462D7]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:47926
   [junit4]   2> 1664092 INFO  (jetty-launcher-3193-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1664093 INFO  (jetty-launcher-3193-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4b8d3ad3{/solr,null,AVAILABLE}
   [junit4]   2> 1664094 INFO  (jetty-launcher-3193-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3c7e31e4{HTTP/1.1,[http/1.1]}{127.0.0.1:36701}
   [junit4]   2> 1664094 INFO  (jetty-launcher-3193-thread-1) [    ] o.e.j.s.Server Started @1666014ms
   [junit4]   2> 1664094 INFO  (jetty-launcher-3193-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=36701}
   [junit4]   2> 1664094 ERROR (jetty-launcher-3193-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1664094 INFO  (jetty-launcher-3193-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1664094 INFO  (jetty-launcher-3193-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1664094 INFO  (jetty-launcher-3193-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1664094 INFO  (jetty-launcher-3193-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-12T12:40:49.789Z
   [junit4]   2> 1664096 INFO  (jetty-launcher-3193-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1664096 INFO  (jetty-launcher-3193-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@32659aa7{/solr,null,AVAILABLE}
   [junit4]   2> 1664097 INFO  (jetty-launcher-3193-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@396d5549{HTTP/1.1,[http/1.1]}{127.0.0.1:56601}
   [junit4]   2> 1664097 INFO  (jetty-launcher-3193-thread-2) [    ] o.e.j.s.Server Started @1666017ms
   [junit4]   2> 1664097 INFO  (jetty-launcher-3193-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=56601}
   [junit4]   2> 1664097 ERROR (jetty-launcher-3193-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1664097 INFO  (jetty-launcher-3193-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1664097 INFO  (jetty-launcher-3193-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1664097 INFO  (jetty-launcher-3193-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1664097 INFO  (jetty-launcher-3193-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-12T12:40:49.792Z
   [junit4]   2> 1664110 INFO  (jetty-launcher-3193-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1664110 INFO  (jetty-launcher-3193-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1664121 INFO  (jetty-launcher-3193-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:47926/solr
   [junit4]   2> 1664124 INFO  (jetty-launcher-3193-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:47926/solr
   [junit4]   2> 1664342 INFO  (jetty-launcher-3193-thread-2) [n:127.0.0.1:56601_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1664359 INFO  (jetty-launcher-3193-thread-2) [n:127.0.0.1:56601_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56601_solr
   [junit4]   2> 1664361 INFO  (jetty-launcher-3193-thread-2) [n:127.0.0.1:56601_solr    ] o.a.s.c.Overseer Overseer (id=98991502107017222-127.0.0.1:56601_solr-n_0000000000) starting
   [junit4]   2> 1664414 INFO  (jetty-launcher-3193-thread-2) [n:127.0.0.1:56601_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56601_solr
   [junit4]   2> 1664419 INFO  (OverseerStateUpdate-98991502107017222-127.0.0.1:56601_solr-n_0000000000) [n:127.0.0.1:56601_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1664461 INFO  (jetty-launcher-3193-thread-2) [n:127.0.0.1:56601_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 1664466 WARN  (jetty-launcher-3193-thread-2) [n:127.0.0.1:56601_solr    ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
   [junit4]   2> 1664468 INFO  (jetty-launcher-3193-thread-2) [n:127.0.0.1:56601_solr    ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
   [junit4]   2> 1664474 INFO  (jetty-launcher-3193-thread-2) [n:127.0.0.1:56601_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 1664481 INFO  (jetty-launcher-3193-thread-2-EventThread) [n:127.0.0.1:56601_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 1664489 INFO  (jetty-launcher-3193-thread-1) [n:127.0.0.1:36701_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1664492 INFO  (jetty-launcher-3193-thread-1) [n:127.0.0.1:36701_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1664494 INFO  (jetty-launcher-3193-thread-1) [n:127.0.0.1:36701_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36701_solr
   [junit4]   2> 1664497 INFO  (zkCallback-3205-thread-1-processing-n:127.0.0.1:56601_solr) [n:127.0.0.1:56601_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1664497 INFO  (zkCallback-3204-thread-1-processing-n:127.0.0.1:36701_solr) [n:127.0.0.1:36701_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1664592 INFO  (jetty-launcher-3193-thread-1) [n:127.0.0.1:36701_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 1664592 WARN  (jetty-launcher-3193-thread-1) [n:127.0.0.1:36701_solr    ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
   [junit4]   2> 1664592 INFO  (jetty-launcher-3193-thread-1) [n:127.0.0.1:36701_solr    ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
   [junit4]   2> 1664593 INFO  (jetty-launcher-3193-thread-1) [n:127.0.0.1:36701_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 1664597 INFO  (jetty-launcher-3193-thread-1-EventThread) [n:127.0.0.1:36701_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 1664615 WARN  (jetty-launcher-3193-thread-1) [n:127.0.0.1:36701_solr    ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
   [junit4]   2> 1664655 INFO  (jetty-launcher-3193-thread-1) [n:127.0.0.1:36701_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36701.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f8690b7
   [junit4]   2> 1664661 INFO  (jetty-launcher-3193-thread-1) [n:127.0.0.1:36701_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36701.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f8690b7
   [junit4]   2> 1664661 INFO  (jetty-launcher-3193-thread-1) [n:127.0.0.1:36701_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36701.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f8690b7
   [junit4]   2> 1664662 INFO  (jetty-launcher-3193-thread-1) [n:127.0.0.1:36701_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_70AC4789D20462D7-001/tempDir-001/node1/.
   [junit4]   2> 1665510 WARN  (jetty-launcher-3193-thread-2) [n:127.0.0.1:56601_solr    ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
   [junit4]   2> 1665539 INFO  (jetty-launcher-3193-thread-2) [n:127.0.0.1:56601_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56601.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f8690b7
   [junit4]   2> 1665545 INFO  (jetty-launcher-3193-thread-2) [n:127.0.0.1:56601_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56601.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f8690b7
   [junit4]   2> 1665545 INFO  (jetty-launcher-3193-thread-2) [n:127.0.0.1:56601_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56601.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f8690b7
   [junit4]   2> 1665546 INFO  (jetty-launcher-3193-thread-2) [n:127.0.0.1:56601_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_70AC4789D20462D7-001/tempDir-001/node2/.
   [junit4]   2> 1665609 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigHosts
   [junit4]   2> 1665623 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigHosts
   [junit4]   2> 1665668 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidRemote
   [junit4]   2> 1665683 WARN  (qtp1939449223-15991) [    ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
   [junit4]   2> 1665724 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidRemote
   [junit4]   2> 1665777 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateAnyHostAnyUser
   [junit4]   2> 1665778 WARN  (qtp1939449223-16008) [    ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
   [junit4]   2> 1665779 INFO  (qtp1939449223-16008) [n:127.0.0.1:36701_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1665779 INFO  (qtp1939449223-16008) [n:127.0.0.1:36701_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1665780 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateAnyHostAnyUser
   [junit4]   2> 1665837 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidProxyUser
   [junit4]   2> 1665838 WARN  (qtp1939449223-16007) [    ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
   [junit4]   2> 1665843 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidProxyUser
   [junit4]   2> 1665899 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testForwarding
   [junit4]   2> 1666421 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[70AC4789D20462D7]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1666422 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[70AC4789D20462D7]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:47926/solr ready
   [junit4]   2> 1666424 INFO  (qtp1939449223-15990) [n:127.0.0.1:36701_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1666428 INFO  (OverseerThreadFactory-5023-thread-1-processing-n:127.0.0.1:56601_solr) [n:127.0.0.1:56601_solr    ] o.a.s.c.CreateCollectionCmd Create collection forwardingCollection
   [junit4]   2> 1666550 INFO  (OverseerStateUpdate-98991502107017222-127.0.0.1:56601_solr-n_0000000000) [n:127.0.0.1:56601_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"forwardingCollection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"forwardingCollection_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:56601/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1666756 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica_n1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1666756 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1666861 INFO  (zkCallback-3205-thread-1-processing-n:127.0.0.1:56601_solr) [n:127.0.0.1:56601_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1666861 INFO  (zkCallback-3205-thread-2-processing-n:127.0.0.1:56601_solr) [n:127.0.0.1:56601_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1667790 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1667817 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.IndexSchema [forwardingCollection_shard1_replica_n1] Schema name=test
   [junit4]   2> 1667884 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1667902 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'forwardingCollection_shard1_replica_n1' using configuration from collection forwardingCollection, trusted=true
   [junit4]   2> 1667902 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_56601.solr.core.forwardingCollection.shard1.replica_n1' (registry 'solr.core.forwardingCollection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f8690b7
   [junit4]   2> 1667902 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1667902 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [[forwardingCollection_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_70AC4789D20462D7-001/tempDir-001/node2/forwardingCollection_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_70AC4789D20462D7-001/tempDir-001/node2/./forwardingCollection_shard1_replica_n1/data/]
   [junit4]   2> 1667904 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1667952 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1667952 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1667953 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1667953 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1667953 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=47.4140625, floorSegmentMB=0.9306640625, forceMergeDeletesPctAllowed=17.502232792442538, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.67114932598997
   [junit4]   2> 1667954 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@114648d0[forwardingCollection_shard1_replica_n1] main]
   [junit4]   2> 1667954 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 1667960 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 1667973 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1667974 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583864037946425344
   [junit4]   2> 1667974 INFO  (searcherExecutor-5028-thread-1-processing-n:127.0.0.1:56601_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 1667974 INFO  (searcherExecutor-5028-thread-1-processing-n:127.0.0.1:56601_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 1667974 INFO  (searcherExecutor-5028-thread-1-processing-n:127.0.0.1:56601_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 1667974 INFO  (searcherExecutor-5028-thread-1-processing-n:127.0.0.1:56601_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 1667974 INFO  (searcherExecutor-5028-thread-1-processing-n:127.0.0.1:56601_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 1667975 INFO  (searcherExecutor-5028-thread-1-processing-n:127.0.0.1:56601_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 1667975 INFO  (searcherExecutor-5028-thread-1-processing-n:127.0.0.1:56601_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 1667975 INFO  (searcherExecutor-5028-thread-1-processing-n:127.0.0.1:56601_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 1667976 INFO  (searcherExecutor-5028-thread-1-processing-n:127.0.0.1:56601_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 1667977 INFO  (searcherExecutor-5028-thread-1-processing-n:127.0.0.1:56601_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica_n1] Registered new searcher Searcher@114648d0[forwardingCollection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1667979 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1667979 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1667979 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56601/solr/forwardingCollection_shard1_replica_n1/
   [junit4]   2> 1667979 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1667979 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:56601/solr/forwardingCollection_shard1_replica_n1/ has no replicas
   [junit4]   2> 1667979 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1667986 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56601/solr/forwardingCollection_shard1_replica_n1/ shard1
   [junit4]   2> 1668089 INFO  (zkCallback-3205-thread-1-processing-n:127.0.0.1:56601_solr) [n:127.0.0.1:56601_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1668089 INFO  (zkCallback-3205-thread-2-processing-n:127.0.0.1:56601_solr) [n:127.0.0.1:56601_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1668137 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1668142 INFO  (qtp580826981-16000) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica_n1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1386
   [junit4]   2> 1668148 INFO  (qtp1939449223-15990) [n:127.0.0.1:36701_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1668246 INFO  (zkCallback-3205-thread-2-processing-n:127.0.0.1:56601_solr) [n:127.0.0.1:56601_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1668246 INFO  (zkCallback-3205-thread-1-processing-n:127.0.0.1:56601_solr) [n:127.0.0.1:56601_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1668432 INFO  (OverseerCollectionConfigSetProcessor-98991502107017222-127.0.0.1:56601_solr-n_0000000000) [n:127.0.0.1:56601_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1669151 INFO  (qtp1939449223-15990) [n:127.0.0.1:36701_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2} status=0 QTime=2728
   [junit4]   2> 1669152 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[70AC4789D20462D7]) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: forwardingCollection failOnTimeout:true timeout (sec):100
   [junit4]   2> 1669154 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[70AC4789D20462D7]) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: forwardingCollection
   [junit4]   2> 1669158 INFO  (qtp1939449223-15985) [n:127.0.0.1:36701_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1669164 INFO  (qtp580826981-15999) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.S.Request [forwardingCollection_shard1_replica_n1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 1669175 INFO  (qtp580826981-16001) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.S.Request [forwardingCollection_shard1_replica_n1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 1669176 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testForwarding
   [junit4]   2> 1669237 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyUnknownRemote
   [junit4]   2> 1669239 WARN  (qtp1939449223-15989) [    ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
   [junit4]   2> 1678984 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[70AC4789D20462D7]-SendThread(127.0.0.1:47926)) [    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 9829ms for sessionid 0x15fb03f26df000b
   [junit4]   2> 1678984 WARN  (jetty-launcher-3193-thread-2-SendThread(127.0.0.1:47926)) [n:127.0.0.1:56601_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 13470ms for sessionid 0x15fb03f26df0007
   [junit4]   2> 1678984 WARN  (jetty-launcher-3193-thread-1-SendThread(127.0.0.1:47926)) [n:127.0.0.1:36701_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 13468ms for sessionid 0x15fb03f26df0008
   [junit4]   2> 1678984 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15fb03f26df0008, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1678984 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15fb03f26df000b, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1678984 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15fb03f26df0007, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1678985 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyUnknownRemote
   [junit4]   2> 1679030 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidGroup
   [junit4]   2> 1679031 WARN  (qtp1939449223-15991) [    ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
   [junit4]   2> 1679056 WARN  (qtp1939449223-15991) [    ] o.a.h.s.ShellBasedUnixGroupsMapping got exception trying to get groups for user bar: id: invalid user name: "bar"
   [junit4]   2> id: invalid user name: "bar"
   [junit4]   2> 
   [junit4]   2> 1679058 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidGroup
   [junit4]   2> 1679091 WARN  (zkCallback-3213-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@738d751 name: ZooKeeperConnection Watcher:127.0.0.1:47926/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1679091 WARN  (zkCallback-3213-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 1679112 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNullProxyUser
   [junit4]   2> 1679113 WARN  (qtp1939449223-16008) [n:127.0.0.1:36701_solr    ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
   [junit4]   2> 1679113 INFO  (qtp1939449223-16008) [n:127.0.0.1:36701_solr    ] o.a.s.s.SolrDispatchFilter Error authenticating
   [junit4]   2> java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:213)
   [junit4]   2> 	at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> 	at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
   [junit4]   2> 	at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:134)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
   [junit4]   2> 	at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:243)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1679113 WARN  (qtp1939449223-16008) [n:127.0.0.1:36701_solr    ] o.e.j.s.ServletHandler /solr/admin/collections
   [junit4]   2> org.apache.solr.common.SolrException: Error during request authentication, 
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:457)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:213)
   [junit4]   2> 	at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> 	at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
   [junit4]   2> 	at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:134)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
   [junit4]   2> 	at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:243)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
   [junit4]   2> 	... 26 more
   [junit4]   2> 1679114 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNullProxyUser
   [junit4]   2> 1679168 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigGroups
   [junit4]   2> 1679169 WARN  (qtp1939449223-15990) [n:127.0.0.1:36701_solr    ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
   [junit4]   2> 1679171 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigGroups
   [junit4]   2> 1679231 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateHost
   [junit4]   2> 1679231 WARN  (qtp1939449223-15985) [n:127.0.0.1:36701_solr    ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
   [junit4]   2> 1679232 INFO  (qtp1939449223-15985) [n:127.0.0.1:36701_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1679232 INFO  (qtp1939449223-15985) [n:127.0.0.1:36701_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1679233 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateHost
   [junit4]   2> 1679282 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateGroup
   [junit4]   2> 1679283 WARN  (qtp1939449223-15987) [    ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
   [junit4]   2> 1679299 INFO  (qtp1939449223-15987) [n:127.0.0.1:36701_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=jenkins&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1679300 INFO  (qtp1939449223-15987) [n:127.0.0.1:36701_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=jenkins&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1679300 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateGroup
   [junit4]   2> 1679347 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyWrongHost
   [junit4]   2> 1679348 WARN  (qtp1939449223-15988) [    ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
   [junit4]   2> 1679369 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[70AC4789D20462D7]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyWrongHost
   [junit4]   2> 1679828 INFO  (jetty-closer-3194-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@3c7e31e4{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1679829 INFO  (jetty-closer-3194-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1145141594
   [junit4]   2> 1679829 INFO  (jetty-closer-3194-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@396d5549{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1679829 INFO  (jetty-closer-3194-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 1679831 INFO  (jetty-closer-3194-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=317679323
   [junit4]   2> 1679831 INFO  (jetty-closer-3194-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@11d3e169: rootName = solr_36701, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@584b0e1c
   [junit4]   2> 1679832 INFO  (jetty-closer-3194-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 1679832 INFO  (jetty-closer-3194-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2cb7b5cb: rootName = solr_56601, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@287aef8f
   [junit4]   2> 1679841 INFO  (jetty-closer-3194-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1679841 INFO  (jetty-closer-3194-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5b8df17e: rootName = solr_36701, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@2932833e
   [junit4]   2> 1679841 INFO  (jetty-closer-3194-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1679841 INFO  (jetty-closer-3194-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@15cadf15: rootName = solr_56601, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@2932833e
   [junit4]   2> 1679846 INFO  (jetty-closer-3194-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1679846 INFO  (jetty-closer-3194-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@459b4a4d: rootName = solr_56601, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@4c46d040
   [junit4]   2> 1679846 INFO  (jetty-closer-3194-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1679846 INFO  (jetty-closer-3194-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2b55a4fd: rootName = solr_36701, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@4c46d040
   [junit4]   2> 1679852 INFO  (jetty-closer-3194-thread-2) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:56601_solr
   [junit4]   2> 1679853 INFO  (jetty-closer-3194-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1679854 INFO  (jetty-closer-3194-thread-1) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:36701_solr
   [junit4]   2> 1679855 INFO  (zkCallback-3205-thread-1-processing-n:127.0.0.1:56601_solr) [n:127.0.0.1:56601_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 1679855 INFO  (zkCallback-3204-thread-1-processing-n:127.0.0.1:36701_solr) [n:127.0.0.1:36701_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 1679856 INFO  (jetty-closer-3194-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1679856 INFO  (coreCloseExecutor-5033-thread-1) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@320155d5
   [junit4]   2> 1679856 INFO  (zkCallback-3205-thread-3-processing-n:127.0.0.1:56601_solr) [n:127.0.0.1:56601_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1679857 INFO  (coreCloseExecutor-5033-thread-1) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.forwardingCollection.shard1.replica_n1, tag=838948309
   [junit4]   2> 1679857 INFO  (coreCloseExecutor-5033-thread-1) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2fe7ba36: rootName = solr_56601, domain = solr.core.forwardingCollection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.forwardingCollection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@3d9abb4b
   [junit4]   2> 1679856 INFO  (zkCallback-3205-thread-2-processing-n:127.0.0.1:56601_solr) [n:127.0.0.1:56601_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1679858 INFO  (zkCallback-3205-thread-4-processing-n:127.0.0.1:56601_solr) [n:127.0.0.1:56601_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 1679859 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15fb03f26df0005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1679860 WARN  (zkCallback-3204-thread-2-processing-n:127.0.0.1:36701_solr) [n:127.0.0.1:36701_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1679861 ERROR (Thread[Thread-5297,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:36701_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 1679884 INFO  (coreCloseExecutor-5033-thread-1) [n:127.0.0.1:56601_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.forwardingCollection.shard1.leader, tag=838948309
   [junit4]   2> 1679885 INFO  (jetty-closer-3194-thread-2) [    ] o.a.s.c.Overseer Overseer (id=98991502107017222-127.0.0.1:56601_solr-n_0000000000) closing
   [junit4]   2> 1679886 INFO  (OverseerStateUpdate-98991502107017222-127.0.0.1:56601_solr-n_0000000000) [n:127.0.0.1:56601_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:56601_solr
   [junit4]   2> 1679886 WARN  (OverseerAutoScalingTriggerThread-98991502107017222-127.0.0.1:56601_solr-n_0000000000) [n:127.0.0.1:56601_solr    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 1679891 ERROR (Thread[Thread-5300,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:56601_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 1680039 INFO  (jetty-closer-3194-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4b8d3ad3{/solr,null,UNAVAILABLE}
   [junit4]   2> 1680739 INFO  (jetty-closer-3194-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@32659aa7{/solr,null,UNAVAILABLE}
   [junit4]   2> 1680744 ERROR (SUITE-TestSolrCloudWithSecureImpersonation-seed#[70AC4789D20462D7]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1680745 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[70AC4789D20462D7]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:47926 47926
   [junit4]   2> 1683883 INFO  (Thread-5274) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:47926 47926
   [junit4]   2> 1683885 WARN  (Thread-5274) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/collections/forwardingCollection/state.json
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 	3	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	3	/solr/collections
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 
   [junit4]   2> 1683917 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[70AC4789D20462D7]-worker) [    ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> Nov 12, 2017 12:41:09 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> Nov 12, 2017 12:41:29 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   [junit4]   2>    1) Thread[id=16042, name=jetty-launcher-3193-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2>         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2>         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2>         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2>         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> Nov 12, 2017 12:41:29 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=16042, name=jetty-launcher-3193-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2> 1704004 ERROR (jetty-launcher-3193-thread-1-EventThread) [n:127.0.0.1:36701_solr    ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2> 	at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2> 	at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2> 	at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2> 	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> Nov 12, 2017 12:41:29 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=336, maxMBSortInHeap=6.50495929345042, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@191b3b30), locale=en-US, timezone=America/Montserrat
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=1,free=351582728,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [HdfsSyncSliceTest, DeleteStatusTest, TestQueryWrapperFilter, TestFreeTextSuggestions, TestPrepRecovery, TestFastLRUCache, AnalysisAfterCoreReloadTest, CheckHdfsIndexTest, TestGroupingSearch, TestExportWriter, TestConfigSetProperties, UninvertDocValuesMergePolicyTest, TestBlobHandler, IndexSchemaRuntimeFieldTest, TestExpandComponent, SimplePostToolTest, HdfsAutoAddReplicasIntegrationTest, TestFilteredDocIdSet, SchemaVersionSpecificBehaviorTest, SOLR749Test, UtilsToolTest, TestCrossCoreJoin, TestRemoteStreaming, HdfsNNFailoverTest, TestJettySolrRunner, LoggingHandlerTest, ReplicationFactorTest, TestManagedSynonymGraphFilterFactory, DocumentBuilderTest, BooleanFieldTest, TestStressInPlaceUpdates, TestConfigOverlay, SolrCoreMetricManagerTest, ReplicaListTransformerTest, MetricsHandlerTest, TestStandardQParsers, TestBulkSchemaConcurrent, TestUseDocValuesAsStored2, TestPKIAuthenticationPlugin, SecurityConfHandlerTest, TestCharFilters, ZkSolrClientTest, FullHLLTest, LeaderElectionContextKeyTest, TestImpersonationWithHadoopAuth, DistributedFacetPivotSmallAdvancedTest, TestSolrJ, HLLUtilTest, BufferStoreTest, TestSchemaSimilarityResource, TestFunctionQuery, SystemInfoHandlerTest, TestCloudSearcherWarming, CoreAdminCreateDiscoverTest, CdcrUpdateLogTest, TestSizeLimitedDistributedMap, TestDocBasedVersionConstraints, ChaosMonkeySafeLeaderTest, TestSolrDeletionPolicy1, TestWordDelimiterFilterFactory, DistributedQueueTest, EchoParamsTest, LeaderElectionIntegrationTest, VersionInfoTest, TestFieldCache, JavabinLoaderTest, SolrIndexSplitterTest, TestFieldCollectionResource, TestOrdValues, TestTrie, TestQueryUtils, HdfsBasicDistributedZkTest, CollectionsAPIAsyncDistributedZkTest, AnalyticsQueryTest, SolrIndexConfigTest, StatsReloadRaceTest, TestCollectionAPIs, ExitableDirectoryReaderTest, TestRTGBase, DistributedFacetPivotLargeTest, TestStressVersions, NumberUtilsTest, TestRequestStatusCollectionAPI, DocValuesMissingTest, TestLuceneIndexBackCompat, TestBinaryResponseWriter, MoreLikeThisHandlerTest, TestSchemalessBufferedUpdates, SolrSlf4jReporterTest, OverseerStatusTest, OpenCloseCoreStressTest, TestObjectReleaseTracker, TestJsonFacets, TestSQLHandler, QueryEqualityTest, TestLegacyFieldReuse, MBeansHandlerTest, TestSegmentSorting, SharedFSAutoReplicaFailoverTest, TestSuggestSpellingConverter, TestPivotHelperCode, DistributedMLTComponentTest, DocValuesTest, RollingRestartTest, CurrencyFieldTypeTest, SubstringBytesRefFilterTest, TestNumericTerms64, TestElisionMultitermQuery, TestPerFieldSimilarity, MoveReplicaHDFSFailoverTest, HdfsRecoveryZkTest, SpatialHeatmapFacetsTest, TestSubQueryTransformer, TestNumericTokenStream, CdcrVersionReplicationTest, SolrPluginUtilsTest, TestAddFieldRealTimeGet, TestFaceting, SimpleFacetsTest, TestSolrIndexConfig, BJQParserTest, BasicFunctionalityTest, JvmMetricsTest, TestDistributedGrouping, SolrXmlInZkTest, TestStressReorder, DistributedIntervalFacetingTest, CoreAdminHandlerTest, TestSubQueryTransformerDistrib, DOMUtilTest, TestSearcherReuse, TestPointFields, TestPostingsSolrHighlighter, SolrShardReporterTest, TermVectorComponentTest, TestScoreJoinQPScore, DebugComponentTest, TestSolrCloudSnapshots, V2StandaloneTest, TestIndexSearcher, PKIAuthenticationIntegrationTest, TestSolrQueryResponse, TestTlogReplica, TestSolrCLIRunExample, DeleteLastCustomShardedReplicaTest, TestSolrCloudWithSecureImpersonation]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=70AC4789D20462D7 -Dtests.slow=true -Dtests.locale=en-US -Dtests.timezone=America/Montserrat -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J1 | TestSolrCloudWithSecureImpersonation (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   [junit4]    >    1) Thread[id=16042, name=jetty-launcher-3193-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([70AC4789D20462D7]:0)
   [junit4] Completed [235/749 (1!)] on J1 in 40.14s, 12 tests, 1 error <<< FAILURES!

[...truncated 2452 lines...]
   [junit4] Suite: org.apache.solr.client.solrj.impl.CloudSolrClientTest
   [junit4]   2> 153518 INFO  (SUITE-CloudSolrClientTest-seed#[7A6529AF03C59754]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.impl.CloudSolrClientTest_7A6529AF03C59754-001/init-core-data-001
   [junit4]   2> 153519 WARN  (SUITE-CloudSolrClientTest-seed#[7A6529AF03C59754]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=102 numCloses=102
   [junit4]   2> 153519 INFO  (SUITE-CloudSolrClientTest-seed#[7A6529AF03C59754]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 153520 INFO  (SUITE-CloudSolrClientTest-seed#[7A6529AF03C59754]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 153520 INFO  (SUITE-CloudSolrClientTest-seed#[7A6529AF03C59754]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.impl.CloudSolrClientTest_7A6529AF03C59754-001/tempDir-001
   [junit4]   2> 153520 INFO  (SUITE-CloudSolrClientTest-seed#[7A6529AF03C59754]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 153521 INFO  (Thread-65) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 153521 INFO  (Thread-65) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 153522 ERROR (Thread-65) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 153621 INFO  (SUITE-CloudSolrClientTest-seed#[7A6529AF03C59754]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:51153
   [junit4]   2> 153638 INFO  (jetty-launcher-1675-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 153638 INFO  (jetty-launcher-1675-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 153639 INFO  (jetty-launcher-1675-thread-3) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 153644 INFO  (jetty-launcher-1675-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@40542abc{/solr,null,AVAILABLE}
   [junit4]   2> 153648 INFO  (jetty-launcher-1675-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3caa31dd{HTTP/1.1,[http/1.1]}{127.0.0.1:35985}
   [junit4]   2> 153648 INFO  (jetty-launcher-1675-thread-2) [    ] o.e.j.s.Server Started @155950ms
   [junit4]   2> 153648 INFO  (jetty-launcher-1675-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35985}
   [junit4]   2> 153649 ERROR (jetty-launcher-1675-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 153649 INFO  (jetty-launcher-1675-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.0.0
   [junit4]   2> 153649 INFO  (jetty-launcher-1675-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 153649 INFO  (jetty-launcher-1675-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 153649 INFO  (jetty-launcher-1675-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-12T13:30:54.626Z
   [junit4]   2> 153650 INFO  (jetty-launcher-1675-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@70b53919{/solr,null,AVAILABLE}
   [junit4]   2> 153651 INFO  (jetty-launcher-1675-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5953aa76{HTTP/1.1,[http/1.1]}{127.0.0.1:44814}
   [junit4]   2> 153651 INFO  (jetty-launcher-1675-thread-3) [    ] o.e.j.s.Server Started @155953ms
   [junit4]   2> 153651 INFO  (jetty-launcher-1675-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=44814}
   [junit4]   2> 153652 ERROR (jetty-launcher-1675-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 153652 INFO  (jetty-launcher-1675-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.0.0
   [junit4]   2> 153652 INFO  (jetty-launcher-1675-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 153652 INFO  (jetty-launcher-1675-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 153653 INFO  (jetty-launcher-1675-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-12T13:30:54.629Z
   [junit4]   2> 153655 INFO  (jetty-launcher-1675-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@151ecf32{/solr,null,AVAILABLE}
   [junit4]   2> 153656 INFO  (jetty-launcher-1675-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1c8767d{HTTP/1.1,[http/1.1]}{127.0.0.1:38895}
   [junit4]   2> 153656 INFO  (jetty-launcher-1675-thread-1) [    ] o.e.j.s.Server Started @155957ms
   [junit4]   2> 153656 INFO  (jetty-launcher-1675-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38895}
   [junit4]   2> 153656 ERROR (jetty-launcher-1675-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 153657 IN

[...truncated too long message...]

ing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@48e55465: rootName = solr_44814, domain = solr.core.localShardsTestColl.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.localShardsTestColl.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@31163d73
   [junit4]   2> 187093 INFO  (coreCloseExecutor-403-thread-4) [n:127.0.0.1:44814_solr c:localShardsTestColl s:shard1 r:core_node3 x:localShardsTestColl_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.localShardsTestColl.shard1.leader, tag=1688895948
   [junit4]   2> 187093 INFO  (coreCloseExecutor-403-thread-5) [n:127.0.0.1:44814_solr c:foo s:shard1 r:core_node5 x:foo_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.foo.shard1.replica_n2, tag=595700355
   [junit4]   2> 187093 INFO  (coreCloseExecutor-403-thread-5) [n:127.0.0.1:44814_solr c:foo s:shard1 r:core_node5 x:foo_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@280729a9: rootName = solr_44814, domain = solr.core.foo.shard1.replica_n2, service url = null, agent id = null] for registry solr.core.foo.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@7bbcfa49
   [junit4]   2> 187099 INFO  (coreCloseExecutor-403-thread-5) [n:127.0.0.1:44814_solr c:foo s:shard1 r:core_node5 x:foo_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.foo.shard1.leader, tag=595700355
   [junit4]   2> 187099 INFO  (coreCloseExecutor-403-thread-6) [n:127.0.0.1:44814_solr c:multicollection2 s:shard1 r:core_node3 x:multicollection2_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.multicollection2.shard1.replica_n1, tag=830145788
   [junit4]   2> 187099 INFO  (coreCloseExecutor-403-thread-6) [n:127.0.0.1:44814_solr c:multicollection2 s:shard1 r:core_node3 x:multicollection2_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6a35233b: rootName = solr_44814, domain = solr.core.multicollection2.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.multicollection2.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@7cad393f
   [junit4]   2> 187107 INFO  (coreCloseExecutor-403-thread-6) [n:127.0.0.1:44814_solr c:multicollection2 s:shard1 r:core_node3 x:multicollection2_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.multicollection2.shard1.leader, tag=830145788
   [junit4]   2> 187107 INFO  (coreCloseExecutor-403-thread-7) [n:127.0.0.1:44814_solr c:2nd_collection s:shard2 r:core_node4 x:2nd_collection_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.2nd_collection.shard2.replica_n2, tag=1832337895
   [junit4]   2> 187107 INFO  (coreCloseExecutor-403-thread-7) [n:127.0.0.1:44814_solr c:2nd_collection s:shard2 r:core_node4 x:2nd_collection_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3097ac10: rootName = solr_44814, domain = solr.core.2nd_collection.shard2.replica_n2, service url = null, agent id = null] for registry solr.core.2nd_collection.shard2.replica_n2 / com.codahale.metrics.MetricRegistry@2698080c
   [junit4]   2> 187112 INFO  (coreCloseExecutor-403-thread-7) [n:127.0.0.1:44814_solr c:2nd_collection s:shard2 r:core_node4 x:2nd_collection_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.2nd_collection.shard2.leader, tag=1832337895
   [junit4]   2> 187117 INFO  (jetty-closer-1676-thread-3) [    ] o.a.s.c.Overseer Overseer (id=98991699034046470-127.0.0.1:44814_solr-n_0000000000) closing
   [junit4]   2> 187117 INFO  (OverseerStateUpdate-98991699034046470-127.0.0.1:44814_solr-n_0000000000) [n:127.0.0.1:44814_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:44814_solr
   [junit4]   2> 187119 WARN  (OverseerAutoScalingTriggerThread-98991699034046470-127.0.0.1:44814_solr-n_0000000000) [n:127.0.0.1:44814_solr    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 189536 WARN  (zkCallback-1688-thread-4-processing-n:127.0.0.1:44814_solr) [n:127.0.0.1:44814_solr c:localShardsTestColl s:shard3 r:core_node16 x:localShardsTestColl_shard3_replica_n8] o.a.s.c.SyncStrategy Closed, skipping sync up.
   [junit4]   2> 189537 INFO  (zkCallback-1688-thread-4-processing-n:127.0.0.1:44814_solr) [n:127.0.0.1:44814_solr c:localShardsTestColl s:shard3 r:core_node16 x:localShardsTestColl_shard3_replica_n8] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 189537 INFO  (zkCallback-1688-thread-4-processing-n:127.0.0.1:44814_solr) [n:127.0.0.1:44814_solr c:localShardsTestColl s:shard3 r:core_node16 x:localShardsTestColl_shard3_replica_n8] o.a.s.c.SolrCore [localShardsTestColl_shard3_replica_n8]  CLOSING SolrCore org.apache.solr.core.SolrCore@876b479
   [junit4]   2> 189537 INFO  (zkCallback-1688-thread-4-processing-n:127.0.0.1:44814_solr) [n:127.0.0.1:44814_solr c:localShardsTestColl s:shard3 r:core_node16 x:localShardsTestColl_shard3_replica_n8] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.localShardsTestColl.shard3.replica_n8, tag=141997177
   [junit4]   2> 189537 INFO  (zkCallback-1688-thread-4-processing-n:127.0.0.1:44814_solr) [n:127.0.0.1:44814_solr c:localShardsTestColl s:shard3 r:core_node16 x:localShardsTestColl_shard3_replica_n8] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1602f7a9: rootName = solr_44814, domain = solr.core.localShardsTestColl.shard3.replica_n8, service url = null, agent id = null] for registry solr.core.localShardsTestColl.shard3.replica_n8 / com.codahale.metrics.MetricRegistry@7ce398e3
   [junit4]   2> 189549 INFO  (zkCallback-1688-thread-4-processing-n:127.0.0.1:44814_solr) [n:127.0.0.1:44814_solr c:localShardsTestColl s:shard3 r:core_node16 x:localShardsTestColl_shard3_replica_n8] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.localShardsTestColl.shard3.leader, tag=141997177
   [junit4]   2> 189552 INFO  (jetty-closer-1676-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@70b53919{/solr,null,UNAVAILABLE}
   [junit4]   2> 189555 ERROR (SUITE-CloudSolrClientTest-seed#[7A6529AF03C59754]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 189556 INFO  (SUITE-CloudSolrClientTest-seed#[7A6529AF03C59754]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:51153 51153
   [junit4]   2> 193130 INFO  (Thread-65) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:51153 51153
   [junit4]   2> 193134 WARN  (Thread-65) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	8	/solr/clusterprops.json
   [junit4]   2> 	8	/solr/aliases.json
   [junit4]   2> 	3	/solr/security.json
   [junit4]   2> 	3	/solr/configs/conf
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	27	/solr/collections/localShardsTestColl/state.json
   [junit4]   2> 	21	/solr/collections/foo/state.json
   [junit4]   2> 	16	/solr/collections/stale_state_test_col/state.json
   [junit4]   2> 	12	/solr/collections/collection1/state.json
   [junit4]   2> 	12	/solr/collections/2nd_collection/state.json
   [junit4]   2> 	12	/solr/collections/multicollection2/state.json
   [junit4]   2> 	12	/solr/collections/multicollection1/state.json
   [junit4]   2> 	10	/solr/collections/nemesis/state.json
   [junit4]   2> 	8	/solr/clusterstate.json
   [junit4]   2> 	6	/solr/collections/overwrite/state.json
   [junit4]   2> 	2	/solr/collections/localShardsTestColl/leader_elect/shard2/election/98991699034046470-core_node13-n_0000000000
   [junit4]   2> 	2	/solr/overseer_elect/election/98991699034046470-127.0.0.1:44814_solr-n_0000000000
   [junit4]   2> 	2	/solr/collections/foo/leader_elect/shard1/election/98991699034046473-core_node3-n_0000000000
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	8	/solr/live_nodes
   [junit4]   2> 	8	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.impl.CloudSolrClientTest_7A6529AF03C59754-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {multiDefault=FSTOrd50, a_t=FSTOrd50, title_s=FSTOrd50, id=PostingsFormat(name=Memory), text=PostingsFormat(name=MockRandom)}, docValues:{_version_=DocValuesFormat(name=Lucene70), multiDefault=DocValuesFormat(name=Direct), title_s=DocValuesFormat(name=Direct), intDefault=DocValuesFormat(name=Lucene70), id=DocValuesFormat(name=Asserting), timestamp=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=1418, maxMBSortInHeap=6.6709383293024445, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@a2ef43a), locale=pl-PL, timezone=America/Creston
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=1,free=163624176,total=367001600
   [junit4]   2> NOTE: All tests run in this JVM: [SquareRootEvaluatorTest, SolrPingTest, SolrZkClientTest, TangentEvaluatorTest, StreamExpressionTest, SolrParamTest, AndEvaluatorTest, JsonValidatorTest, SolrExampleBinaryTest, OrEvaluatorTest, QueryResponseTest, CloudSolrClientTest]
   [junit4] Completed [115/146 (1!)] on J1 in 39.67s, 16 tests, 1 error <<< FAILURES!

[...truncated 46412 lines...]

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1531 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1531/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

2 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.uninverting.TestNumericTerms64

Error Message:
The test or suite printed 13416 bytes to stdout and stderr, even though the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it completely with @SuppressSysoutChecks or run with -Dtests.verbose=true

Stack Trace:
java.lang.AssertionError: The test or suite printed 13416 bytes to stdout and stderr, even though the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it completely with @SuppressSysoutChecks or run with -Dtests.verbose=true
	at __randomizedtesting.SeedInfo.seed([A1BAB4070756F514]:0)
	at org.apache.lucene.util.TestRuleLimitSysouts.afterIfSuccessful(TestRuleLimitSysouts.java:211)
	at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterIfSuccessful(TestRuleAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:37)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:     1) Thread[id=21731, name=jetty-launcher-4005-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)    2) Thread[id=21719, name=jetty-launcher-4005-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   1) Thread[id=21731, name=jetty-launcher-4005-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   2) Thread[id=21719, name=jetty-launcher-4005-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
	at __randomizedtesting.SeedInfo.seed([A1BAB4070756F514]:0)




Build Log:
[...truncated 12650 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_A1BAB4070756F514-001/init-core-data-001
   [junit4]   2> 2059936 WARN  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[A1BAB4070756F514]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=7 numCloses=7
   [junit4]   2> 2059936 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[A1BAB4070756F514]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 2059938 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[A1BAB4070756F514]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2059938 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[A1BAB4070756F514]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2060031 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[A1BAB4070756F514]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_A1BAB4070756F514-001/tempDir-001
   [junit4]   2> 2060031 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[A1BAB4070756F514]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2060031 INFO  (Thread-7298) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2060031 INFO  (Thread-7298) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2060039 ERROR (Thread-7298) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2060161 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[A1BAB4070756F514]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:48531
   [junit4]   2> 2060241 INFO  (jetty-launcher-4005-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2060242 INFO  (jetty-launcher-4005-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7c30f282{/solr,null,AVAILABLE}
   [junit4]   2> 2060242 INFO  (jetty-launcher-4005-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6066a25d{HTTP/1.1,[http/1.1]}{127.0.0.1:45596}
   [junit4]   2> 2060242 INFO  (jetty-launcher-4005-thread-1) [    ] o.e.j.s.Server Started @2062520ms
   [junit4]   2> 2060243 INFO  (jetty-launcher-4005-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45596}
   [junit4]   2> 2060243 ERROR (jetty-launcher-4005-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2060243 INFO  (jetty-launcher-4005-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.0.0
   [junit4]   2> 2060243 INFO  (jetty-launcher-4005-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2060243 INFO  (jetty-launcher-4005-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2060243 INFO  (jetty-launcher-4005-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-12T23:07:45.363Z
   [junit4]   2> 2060246 INFO  (jetty-launcher-4005-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2060247 INFO  (jetty-launcher-4005-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6a64dc4c{/solr,null,AVAILABLE}
   [junit4]   2> 2060248 INFO  (jetty-launcher-4005-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@73bb7300{HTTP/1.1,[http/1.1]}{127.0.0.1:49748}
   [junit4]   2> 2060248 INFO  (jetty-launcher-4005-thread-2) [    ] o.e.j.s.Server Started @2062525ms
   [junit4]   2> 2060248 INFO  (jetty-launcher-4005-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=49748}
   [junit4]   2> 2060248 ERROR (jetty-launcher-4005-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2060248 INFO  (jetty-launcher-4005-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.0.0
   [junit4]   2> 2060248 INFO  (jetty-launcher-4005-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2060249 INFO  (jetty-launcher-4005-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2060249 INFO  (jetty-launcher-4005-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-12T23:07:45.369Z
   [junit4]   2> 2060285 INFO  (jetty-launcher-4005-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2060298 INFO  (jetty-launcher-4005-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2060312 INFO  (jetty-launcher-4005-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48531/solr
   [junit4]   2> 2060316 INFO  (jetty-launcher-4005-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48531/solr
   [junit4]   2> 2060435 INFO  (jetty-launcher-4005-thread-1) [n:127.0.0.1:45596_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2060437 INFO  (jetty-launcher-4005-thread-1) [n:127.0.0.1:45596_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45596_solr
   [junit4]   2> 2060447 INFO  (jetty-launcher-4005-thread-1) [n:127.0.0.1:45596_solr    ] o.a.s.c.Overseer Overseer (id=98993967277801478-127.0.0.1:45596_solr-n_0000000000) starting
   [junit4]   2> 2060466 INFO  (jetty-launcher-4005-thread-1) [n:127.0.0.1:45596_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45596_solr
   [junit4]   2> 2060617 INFO  (jetty-launcher-4005-thread-2) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2060619 INFO  (jetty-launcher-4005-thread-2) [n:127.0.0.1:49748_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2060630 INFO  (jetty-launcher-4005-thread-2) [n:127.0.0.1:49748_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49748_solr
   [junit4]   2> 2060655 INFO  (zkCallback-4016-thread-1-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2060656 INFO  (OverseerStateUpdate-98993967277801478-127.0.0.1:45596_solr-n_0000000000) [n:127.0.0.1:45596_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2060749 INFO  (jetty-launcher-4005-thread-2) [n:127.0.0.1:49748_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 2060749 WARN  (jetty-launcher-4005-thread-2) [n:127.0.0.1:49748_solr    ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
   [junit4]   2> 2060749 INFO  (jetty-launcher-4005-thread-2) [n:127.0.0.1:49748_solr    ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
   [junit4]   2> 2060776 INFO  (jetty-launcher-4005-thread-2) [n:127.0.0.1:49748_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 2060797 INFO  (jetty-launcher-4005-thread-2-EventThread) [n:127.0.0.1:49748_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 2060857 WARN  (jetty-launcher-4005-thread-2) [n:127.0.0.1:49748_solr    ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
   [junit4]   2> 2060942 INFO  (jetty-launcher-4005-thread-2) [n:127.0.0.1:49748_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49748.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4e3966c0
   [junit4]   2> 2060950 INFO  (jetty-launcher-4005-thread-2) [n:127.0.0.1:49748_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49748.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4e3966c0
   [junit4]   2> 2060952 INFO  (jetty-launcher-4005-thread-2) [n:127.0.0.1:49748_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49748.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4e3966c0
   [junit4]   2> 2060953 INFO  (jetty-launcher-4005-thread-2) [n:127.0.0.1:49748_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_A1BAB4070756F514-001/tempDir-001/node2/.
   [junit4]   2> 2060997 INFO  (jetty-launcher-4005-thread-1) [n:127.0.0.1:45596_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 2060998 WARN  (jetty-launcher-4005-thread-1) [n:127.0.0.1:45596_solr    ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
   [junit4]   2> 2060998 INFO  (jetty-launcher-4005-thread-1) [n:127.0.0.1:45596_solr    ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
   [junit4]   2> 2061016 INFO  (jetty-launcher-4005-thread-1) [n:127.0.0.1:45596_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 2061029 INFO  (jetty-launcher-4005-thread-1-EventThread) [n:127.0.0.1:45596_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 2061231 WARN  (jetty-launcher-4005-thread-1) [n:127.0.0.1:45596_solr    ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
   [junit4]   2> 2061334 INFO  (jetty-launcher-4005-thread-1) [n:127.0.0.1:45596_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45596.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4e3966c0
   [junit4]   2> 2061344 INFO  (jetty-launcher-4005-thread-1) [n:127.0.0.1:45596_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45596.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4e3966c0
   [junit4]   2> 2061345 INFO  (jetty-launcher-4005-thread-1) [n:127.0.0.1:45596_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45596.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4e3966c0
   [junit4]   2> 2061346 INFO  (jetty-launcher-4005-thread-1) [n:127.0.0.1:45596_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_A1BAB4070756F514-001/tempDir-001/node1/.
   [junit4]   2> 2061866 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidGroup
   [junit4]   2> 2062008 WARN  (qtp526720809-21683) [    ] o.a.h.s.ShellBasedUnixGroupsMapping got exception trying to get groups for user bar: id: invalid user name: "bar"
   [junit4]   2> id: invalid user name: "bar"
   [junit4]   2> 
   [junit4]   2> 2062012 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[A1BAB4070756F514]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bogusGroup&p=bogusGroup&t=test&e=1510564067015&s=PmZPQRnjZzZy7JnwGVc6dLKETS0="; Path=/; Domain=127.0.0.1; Expires=lun, 13-nov-2017 09:07:47 GMT; HttpOnly". Invalid 'expires' attribute: lun, 13-nov-2017 09:07:47 GMT
   [junit4]   2> 2062012 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidGroup
   [junit4]   2> 2062679 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigHosts
   [junit4]   2> 2062734 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[A1BAB4070756F514]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noHosts&p=noHosts&t=test&e=1510564067804&s=3q2jZ+Xp6ybh7DiiEsU7tWKjFvo="; Path=/; Domain=127.0.0.1; Expires=lun, 13-nov-2017 09:07:47 GMT; HttpOnly". Invalid 'expires' attribute: lun, 13-nov-2017 09:07:47 GMT
   [junit4]   2> 2062734 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigHosts
   [junit4]   2> 2063403 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigGroups
   [junit4]   2> 2063416 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[A1BAB4070756F514]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noGroups&p=noGroups&t=test&e=1510564068530&s=7a9bKeqbDXa/3qN+s8QC/DuYVcQ="; Path=/; Domain=127.0.0.1; Expires=lun, 13-nov-2017 09:07:48 GMT; HttpOnly". Invalid 'expires' attribute: lun, 13-nov-2017 09:07:48 GMT
   [junit4]   2> 2063416 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigGroups
   [junit4]   2> 2064029 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyUnknownRemote
   [junit4]   2> 2082001 WARN  (jetty-launcher-4005-thread-1-SendThread(127.0.0.1:48531)) [n:127.0.0.1:45596_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 20723ms for sessionid 0x15fb27d1e800008
   [junit4]   2> 2082002 WARN  (jetty-launcher-4005-thread-1-SendThread(127.0.0.1:48531)) [n:127.0.0.1:45596_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 19523ms for sessionid 0x15fb27d1e800006
   [junit4]   2> 2082002 WARN  (jetty-launcher-4005-thread-2-SendThread(127.0.0.1:48531)) [n:127.0.0.1:49748_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 21048ms for sessionid 0x15fb27d1e800005
   [junit4]   2> 2082002 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15fb27d1e800006, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2082004 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[A1BAB4070756F514]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=test&e=1510564069152&s=wkhFoxOth5lcPe4H6jeFbxk9X/M="; Path=/; Domain=127.0.0.1; Expires=lun, 13-nov-2017 09:07:49 GMT; HttpOnly". Invalid 'expires' attribute: lun, 13-nov-2017 09:07:49 GMT
   [junit4]   2> 2082004 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyUnknownRemote
   [junit4]   2> 2082147 WARN  (zkCallback-4016-thread-3-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@55b7038b name: ZooKeeperConnection Watcher:127.0.0.1:48531/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 2082147 WARN  (zkCallback-4016-thread-3-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 2082149 WARN  (zkCallback-4017-thread-2-processing-n:127.0.0.1:45596_solr) [n:127.0.0.1:45596_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5f529186 name: ZooKeeperConnection Watcher:127.0.0.1:48531/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 2082149 WARN  (zkCallback-4017-thread-2-processing-n:127.0.0.1:45596_solr) [n:127.0.0.1:45596_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 2082420 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNullProxyUser
   [junit4]   2> 2082422 INFO  (qtp526720809-21678) [    ] o.a.s.s.SolrDispatchFilter Error authenticating
   [junit4]   2> java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:213)
   [junit4]   2> 	at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> 	at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
   [junit4]   2> 	at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:134)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
   [junit4]   2> 	at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:243)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2082422 WARN  (qtp526720809-21678) [    ] o.e.j.s.ServletHandler /solr/admin/collections
   [junit4]   2> org.apache.solr.common.SolrException: Error during request authentication, 
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:457)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:213)
   [junit4]   2> 	at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> 	at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
   [junit4]   2> 	at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:134)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
   [junit4]   2> 	at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:243)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
   [junit4]   2> 	... 26 more
   [junit4]   2> 2082424 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNullProxyUser
   [junit4]   2> 2082684 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidRemote
   [junit4]   2> 2082788 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[A1BAB4070756F514]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=test&e=1510564087806&s=JBemNH9hJWZW1uVgySlpDAOcNp8="; Path=/; Domain=127.0.0.1; Expires=lun, 13-nov-2017 09:08:07 GMT; HttpOnly". Invalid 'expires' attribute: lun, 13-nov-2017 09:08:07 GMT
   [junit4]   2> 2082788 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidRemote
   [junit4]   2> 2083054 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyWrongHost
   [junit4]   2> 2086010 WARN  (jetty-launcher-4005-thread-2-SendThread(127.0.0.1:48531)) [n:127.0.0.1:49748_solr    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15fb27d1e800007 has expired
   [junit4]   2> 2086014 WARN  (jetty-launcher-4005-thread-1-SendThread(127.0.0.1:48531)) [n:127.0.0.1:45596_solr    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15fb27d1e800008 has expired
   [junit4]   2> 2086015 WARN  (jetty-launcher-4005-thread-2-SendThread(127.0.0.1:48531)) [n:127.0.0.1:49748_solr    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15fb27d1e800005 has expired
   [junit4]   2> 2086015 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[A1BAB4070756F514]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=wrongHost&p=wrongHost&t=test&e=1510564088175&s=/tOAPQnSxsH5T6rP5NjEbybMlVw="; Path=/; Domain=127.0.0.1; Expires=lun, 13-nov-2017 09:08:08 GMT; HttpOnly". Invalid 'expires' attribute: lun, 13-nov-2017 09:08:08 GMT
   [junit4]   2> 2086015 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyWrongHost
   [junit4]   2> 2086053 WARN  (zkCallback-4016-thread-8-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@55b7038b name: ZooKeeperConnection Watcher:127.0.0.1:48531/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
   [junit4]   2> 2086061 WARN  (zkCallback-4016-thread-8-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
   [junit4]   2> 2086078 INFO  (zkCallback-4016-thread-8-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2086085 WARN  (zkCallback-4016-thread-8-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
   [junit4]   2> 2086087 INFO  (zkCallback-4016-thread-8-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
   [junit4]   2> 2086087 INFO  (zkCallback-4016-thread-8-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
   [junit4]   2> 2086088 INFO  (zkCallback-4017-thread-10-processing-n:127.0.0.1:45596_solr) [n:127.0.0.1:45596_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 2086156 INFO  (zkCallback-4016-thread-8-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 2086293 INFO  (zkCallback-4016-thread-8-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49748_solr
   [junit4]   2> 2086297 INFO  (zkCallback-4017-thread-10-processing-n:127.0.0.1:45596_solr) [n:127.0.0.1:45596_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2086297 INFO  (zkCallback-4016-thread-8-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.DefaultConnectionStrategy Reconnected to ZooKeeper
   [junit4]   2> 2086312 INFO  (zkCallback-4016-thread-8-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.ConnectionManager Connected:true
   [junit4]   2> 2086314 INFO  (zkCallback-4016-thread-8-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2086515 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateGroup
   [junit4]   2> 2086569 INFO  (qtp526720809-21683) [n:127.0.0.1:49748_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=jenkins&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2086569 INFO  (qtp526720809-21683) [n:127.0.0.1:49748_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=jenkins&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2086570 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[A1BAB4070756F514]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostUsersGroup&p=anyHostUsersGroup&t=test&e=1510564091636&s=sqmU12w9xKYR9AZhv6XcJrNsYlg="; Path=/; Domain=127.0.0.1; Expires=lun, 13-nov-2017 09:08:11 GMT; HttpOnly". Invalid 'expires' attribute: lun, 13-nov-2017 09:08:11 GMT
   [junit4]   2> 2086570 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateGroup
   [junit4]   2> 2087096 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateAnyHostAnyUser
   [junit4]   2> 2087101 INFO  (qtp526720809-21689) [n:127.0.0.1:49748_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2087101 INFO  (qtp526720809-21689) [n:127.0.0.1:49748_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2087102 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[A1BAB4070756F514]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostAnyUser&p=anyHostAnyUser&t=test&e=1510564092221&s=lKDB1Q4Z1NfOUKxHw1UlgmxfWQs="; Path=/; Domain=127.0.0.1; Expires=lun, 13-nov-2017 09:08:12 GMT; HttpOnly". Invalid 'expires' attribute: lun, 13-nov-2017 09:08:12 GMT
   [junit4]   2> 2087102 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateAnyHostAnyUser
   [junit4]   2> 2087544 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidProxyUser
   [junit4]   2> 2087552 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[A1BAB4070756F514]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bar&p=bar&t=test&e=1510564092668&s=be7TISCaHkqB9w3tyhG8JrC5CFw="; Path=/; Domain=127.0.0.1; Expires=lun, 13-nov-2017 09:08:12 GMT; HttpOnly". Invalid 'expires' attribute: lun, 13-nov-2017 09:08:12 GMT
   [junit4]   2> 2087552 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidProxyUser
   [junit4]   2> 2088077 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testForwarding
   [junit4]   2> 2089316 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[A1BAB4070756F514]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2089318 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[A1BAB4070756F514]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48531/solr ready
   [junit4]   2> 2089328 INFO  (qtp936311320-21671) [n:127.0.0.1:45596_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2089334 INFO  (OverseerThreadFactory-6077-thread-1-processing-n:127.0.0.1:45596_solr) [n:127.0.0.1:45596_solr    ] o.a.s.c.CreateCollectionCmd Create collection forwardingCollection
   [junit4]   2> 2089449 INFO  (OverseerStateUpdate-98993967277801478-127.0.0.1:45596_solr-n_0000000000) [n:127.0.0.1:45596_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"forwardingCollection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"forwardingCollection_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:49748/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 2089654 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica_n1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2089764 INFO  (zkCallback-4016-thread-8-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2089764 INFO  (zkCallback-4016-thread-5-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2090689 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 2090713 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.IndexSchema [forwardingCollection_shard1_replica_n1] Schema name=test
   [junit4]   2> 2090846 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2090868 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'forwardingCollection_shard1_replica_n1' using configuration from collection forwardingCollection, trusted=true
   [junit4]   2> 2090868 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49748.solr.core.forwardingCollection.shard1.replica_n1' (registry 'solr.core.forwardingCollection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4e3966c0
   [junit4]   2> 2090868 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2090868 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [[forwardingCollection_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_A1BAB4070756F514-001/tempDir-001/node2/forwardingCollection_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_A1BAB4070756F514-001/tempDir-001/node2/./forwardingCollection_shard1_replica_n1/data/]
   [junit4]   2> 2090870 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=46.7802734375, floorSegmentMB=1.8486328125, forceMergeDeletesPctAllowed=12.376082375996976, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 2090939 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2090939 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2090940 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2090940 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2090942 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=35, maxMergeAtOnceExplicit=12, maxMergedSegmentMB=39.701171875, floorSegmentMB=1.5576171875, forceMergeDeletesPctAllowed=24.07171708388356, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 2090942 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4bef9ebc[forwardingCollection_shard1_replica_n1] main]
   [junit4]   2> 2090949 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 2090953 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 2090976 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2090976 INFO  (searcherExecutor-6082-thread-1-processing-n:127.0.0.1:49748_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 2090976 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583903508891959296
   [junit4]   2> 2090976 INFO  (searcherExecutor-6082-thread-1-processing-n:127.0.0.1:49748_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 2090977 INFO  (searcherExecutor-6082-thread-1-processing-n:127.0.0.1:49748_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 2090977 INFO  (searcherExecutor-6082-thread-1-processing-n:127.0.0.1:49748_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 2090977 INFO  (searcherExecutor-6082-thread-1-processing-n:127.0.0.1:49748_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 2090978 INFO  (searcherExecutor-6082-thread-1-processing-n:127.0.0.1:49748_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 2090978 INFO  (searcherExecutor-6082-thread-1-processing-n:127.0.0.1:49748_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 2090978 INFO  (searcherExecutor-6082-thread-1-processing-n:127.0.0.1:49748_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 2090979 INFO  (searcherExecutor-6082-thread-1-processing-n:127.0.0.1:49748_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 2090981 INFO  (searcherExecutor-6082-thread-1-processing-n:127.0.0.1:49748_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica_n1] Registered new searcher Searcher@4bef9ebc[forwardingCollection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2090985 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2090986 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2090986 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:49748/solr/forwardingCollection_shard1_replica_n1/
   [junit4]   2> 2090986 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2090986 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:49748/solr/forwardingCollection_shard1_replica_n1/ has no replicas
   [junit4]   2> 2090986 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2090994 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:49748/solr/forwardingCollection_shard1_replica_n1/ shard1
   [junit4]   2> 2091098 INFO  (zkCallback-4016-thread-8-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2091098 INFO  (zkCallback-4016-thread-5-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2091147 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2091150 INFO  (qtp526720809-21691) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica_n1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1495
   [junit4]   2> 2091150 WARN  (httpShardExecutor-6069-thread-1-processing-n:127.0.0.1:45596_solr [http:////127.0.0.1:49748//solr] http:////127.0.0.1:49748//solr) [n:127.0.0.1:45596_solr    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=$&p=$&t=test&e=1510564094773&s=SbUouzcHsSzauv58awoF477PXZY="; Path=/; Domain=127.0.0.1; Expires=lun, 13-nov-2017 09:08:14 GMT; HttpOnly". Invalid 'expires' attribute: lun, 13-nov-2017 09:08:14 GMT
   [junit4]   2> 2091157 INFO  (qtp936311320-21671) [n:127.0.0.1:45596_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 2091250 INFO  (zkCallback-4016-thread-8-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2091251 INFO  (zkCallback-4016-thread-5-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2091336 INFO  (OverseerCollectionConfigSetProcessor-98993967277801478-127.0.0.1:45596_solr-n_0000000000) [n:127.0.0.1:45596_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2092168 INFO  (qtp936311320-21671) [n:127.0.0.1:45596_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2} status=0 QTime=2840
   [junit4]   2> 2092168 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[A1BAB4070756F514]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=user&p=user&t=test&e=1510564094448&s=fY4VyGnYeafR43+1Gier0EeQj/I="; Path=/; Domain=127.0.0.1; Expires=lun, 13-nov-2017 09:08:14 GMT; HttpOnly". Invalid 'expires' attribute: lun, 13-nov-2017 09:08:14 GMT
   [junit4]   2> 2092168 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[A1BAB4070756F514]) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: forwardingCollection failOnTimeout:true timeout (sec):100
   [junit4]   2> 2092169 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[A1BAB4070756F514]) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: forwardingCollection
   [junit4]   2> 2092173 INFO  (qtp526720809-21684) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.S.Request [forwardingCollection_shard1_replica_n1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 2092173 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[A1BAB4070756F514]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=user&p=user&t=test&e=1510564097292&s=DcjZg2tSaGmNWiYvFs0yShyniqw="; Path=/; Domain=127.0.0.1; Expires=lun, 13-nov-2017 09:08:17 GMT; HttpOnly". Invalid 'expires' attribute: lun, 13-nov-2017 09:08:17 GMT
   [junit4]   2> 2092180 INFO  (qtp936311320-21676) [n:127.0.0.1:45596_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2092190 INFO  (qtp526720809-21685) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.S.Request [forwardingCollection_shard1_replica_n1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 2092190 WARN  (qtp936311320-21676) [n:127.0.0.1:45596_solr    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=$&p=$&t=test&e=1510564097309&s=ZM9xo63YM7VltZb1bOX6OUUdcI0="; Path=/; Domain=127.0.0.1; Expires=lun, 13-nov-2017 09:08:17 GMT; HttpOnly". Invalid 'expires' attribute: lun, 13-nov-2017 09:08:17 GMT
   [junit4]   2> 2092191 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[A1BAB4070756F514]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=user&p=user&t=test&e=1510564097296&s=vMr8Bas+C3+qxtD0LZmrbgJZnY4="; Path=/; Domain=127.0.0.1; Expires=lun, 13-nov-2017 09:08:17 GMT; HttpOnly". Invalid 'expires' attribute: lun, 13-nov-2017 09:08:17 GMT
   [junit4]   2> 2092191 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[A1BAB4070756F514]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=$&p=$&t=test&e=1510564097309&s=ZM9xo63YM7VltZb1bOX6OUUdcI0="; Path=/; Domain=127.0.0.1; Expires=lun, 13-nov-2017 09:08:17 GMT; HttpOnly". Invalid 'expires' attribute: lun, 13-nov-2017 09:08:17 GMT
   [junit4]   2> 2092192 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testForwarding
   [junit4]   2> 2092571 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateHost
   [junit4]   2> 2092578 INFO  (qtp526720809-21683) [n:127.0.0.1:49748_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2092578 INFO  (qtp526720809-21683) [n:127.0.0.1:49748_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2092579 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[A1BAB4070756F514]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=test&e=1510564097698&s=hiseOvzpI1UdPHyy0EZEQXIQBZM="; Path=/; Domain=127.0.0.1; Expires=lun, 13-nov-2017 09:08:17 GMT; HttpOnly". Invalid 'expires' attribute: lun, 13-nov-2017 09:08:17 GMT
   [junit4]   2> 2092579 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[A1BAB4070756F514]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateHost
   [junit4]   2> 2092582 INFO  (jetty-closer-4006-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@6066a25d{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2092636 INFO  (jetty-closer-4006-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@73bb7300{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2092636 INFO  (jetty-closer-4006-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1470272117
   [junit4]   2> 2092636 INFO  (jetty-closer-4006-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2092636 INFO  (jetty-closer-4006-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4d25be7d: rootName = solr_45596, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@1432e62
   [junit4]   2> 2092651 INFO  (jetty-closer-4006-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1694109168
   [junit4]   2> 2092651 INFO  (jetty-closer-4006-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2092651 INFO  (jetty-closer-4006-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@104b030: rootName = solr_49748, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@5935f3e5
   [junit4]   2> 2092666 INFO  (jetty-closer-4006-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2092666 INFO  (jetty-closer-4006-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3f748e9b: rootName = solr_49748, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@1cb2a9c9
   [junit4]   2> 2092671 INFO  (jetty-closer-4006-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2092671 INFO  (jetty-closer-4006-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@186560d9: rootName = solr_45596, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@1cb2a9c9
   [junit4]   2> 2092680 INFO  (jetty-closer-4006-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2092680 INFO  (jetty-closer-4006-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@55ceff5d: rootName = solr_49748, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@63242420
   [junit4]   2> 2092682 INFO  (jetty-closer-4006-thread-1) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:49748_solr
   [junit4]   2> 2092686 INFO  (jetty-closer-4006-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2092686 INFO  (jetty-closer-4006-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@37dfb4aa: rootName = solr_45596, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@63242420
   [junit4]   2> 2092692 INFO  (jetty-closer-4006-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 2092697 INFO  (jetty-closer-4006-thread-2) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:45596_solr
   [junit4]   2> 2092699 INFO  (jetty-closer-4006-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 2092700 INFO  (jetty-closer-4006-thread-2) [    ] o.a.s.c.Overseer Overseer (id=98993967277801478-127.0.0.1:45596_solr-n_0000000000) closing
   [junit4]   2> 2092700 WARN  (OverseerAutoScalingTriggerThread-98993967277801478-127.0.0.1:45596_solr-n_0000000000) [n:127.0.0.1:45596_solr    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 2092724 INFO  (OverseerStateUpdate-98993967277801478-127.0.0.1:45596_solr-n_0000000000) [n:127.0.0.1:45596_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:45596_solr
   [junit4]   2> 2092726 INFO  (coreCloseExecutor-6088-thread-1) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@3ea971e8
   [junit4]   2> 2092726 INFO  (coreCloseExecutor-6088-thread-1) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.forwardingCollection.shard1.replica_n1, tag=1051292136
   [junit4]   2> 2092726 INFO  (coreCloseExecutor-6088-thread-1) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@58926e6: rootName = solr_49748, domain = solr.core.forwardingCollection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.forwardingCollection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@4441fc89
   [junit4]   2> 2092732 INFO  (zkCallback-4017-thread-10-processing-n:127.0.0.1:45596_solr) [n:127.0.0.1:45596_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 2092736 INFO  (zkCallback-4017-thread-11-processing-n:127.0.0.1:45596_solr) [n:127.0.0.1:45596_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 2092768 INFO  (coreCloseExecutor-6088-thread-1) [n:127.0.0.1:49748_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.forwardingCollection.shard1.leader, tag=1051292136
   [junit4]   2> 2092748 INFO  (zkCallback-4016-thread-3-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:49748_solr
   [junit4]   2> 2092737 INFO  (zkCallback-4016-thread-8-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 2092736 INFO  (zkCallback-4016-thread-5-processing-n:127.0.0.1:49748_solr) [n:127.0.0.1:49748_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 2092796 ERROR (Thread[Thread-7324,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:45596_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 2092804 INFO  (jetty-closer-4006-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7c30f282{/solr,null,UNAVAILABLE}
   [junit4]   2> 2094298 ERROR (Thread[Thread-7321,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:49748_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 2094303 INFO  (jetty-closer-4006-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6a64dc4c{/solr,null,UNAVAILABLE}
   [junit4]   2> 2094307 ERROR (SUITE-TestSolrCloudWithSecureImpersonation-seed#[A1BAB4070756F514]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2094307 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[A1BAB4070756F514]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:48531 48531
   [junit4]   2> 2100858 INFO  (Thread-7298) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:48531 48531
   [junit4]   2> 2100876 WARN  (Thread-7298) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	4	/solr/aliases.json
   [junit4]   2> 	4	/solr/clusterprops.json
   [junit4]   2> 	3	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/collections/forwardingCollection/state.json
   [junit4]   2> 	4	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/overseer_elect/election/98993967277801478-127.0.0.1:45596_solr-n_0000000000
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	4	/solr/collections
   [junit4]   2> 	2	/solr/live_nodes
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 
   [junit4]   2> 2100908 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[A1BAB4070756F514]-worker) [    ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> Nov 12, 2017 11:08:26 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> Nov 12, 2017 11:08:46 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   [junit4]   2>    1) Thread[id=21731, name=jetty-launcher-4005-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2>         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2>         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2>         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2>         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2>    2) Thread[id=21719, name=jetty-launcher-4005-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2>         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2>         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2>         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2>         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> Nov 12, 2017 11:08:46 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=21731, name=jetty-launcher-4005-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2>    2) Thread[id=21719, name=jetty-launcher-4005-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2> 2120993 ERROR (jetty-launcher-4005-thread-1-EventThread) [n:127.0.0.1:45596_solr    ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2> 	at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2> 	at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2> 	at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2> 	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> 2120993 ERROR (jetty-launcher-4005-thread-2-EventThread) [n:127.0.0.1:49748_solr    ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2> 	at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2> 	at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2> 	at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2> 	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> 2120995 WARN  (jetty-launcher-4005-thread-2-EventThread) [n:127.0.0.1:49748_solr    ] o.a.c.ConnectionState Session expired event received
   [junit4]   2> 2120995 WARN  (jetty-launcher-4005-thread-1-EventThread) [n:127.0.0.1:45596_solr    ] o.a.c.ConnectionState Session expired event received
   [junit4]   2> 2120998 WARN  (jetty-launcher-4005-thread-2-SendThread(127.0.0.1:48531)) [n:127.0.0.1:49748_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
   [junit4]   2> Nov 12, 2017 11:08:46 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> 2120998 WARN  (jetty-launcher-4005-thread-1-SendThread(127.0.0.1:48531)) [n:127.0.0.1:45596_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1050, maxMBSortInHeap=5.2166381012088685, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@702e8755), locale=es-ES, timezone=Etc/UTC
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=5,free=95151664,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [TestSchemalessBufferedUpdates, LeaderElectionContextKeyTest, PKIAuthenticationIntegrationTest, TestPointFields, BJQParserTest, CoreAdminHandlerTest, TestInPlaceUpdatesStandalone, DocumentBuilderTest, ZkNodePropsTest, TestCloudSearcherWarming, TestJsonFacets, TestNumericTokenStream, TestSimpleTrackingShardHandler, BigEndianAscendingWordDeserializerTest, SecurityConfHandlerTest, TestDistributedMissingSort, TestClusterStateMutator, EchoParamsTest, SuggestComponentTest, TestNonDefinedSimilarityFactory, TestMissingGroups, TestManagedStopFilterFactory, TestGraphMLResponseWriter, TestExpandComponent, TestBadConfig, TestStressVersions, TestSerializedLuceneMatchVersion, DistributedMLTComponentTest, TestObjectReleaseTracker, DebugComponentTest, TestLuceneMatchVersion, TestRecoveryHdfs, TestSolrCoreParser, TestInfoStreamLogging, ConcurrentDeleteAndCreateCollectionTest, AtomicUpdateProcessorFactoryTest, MigrateRouteKeyTest, NumberUtilsTest, SoftAutoCommitTest, PluginInfoTest, TestShardHandlerFactory, TestReplicaProperties, TestDynamicLoading, SimpleFacetsTest, TestDocumentBuilder, TestFastOutputStream, MBeansHandlerTest, SimplePostToolTest, SOLR749Test, TestDeleteCollectionOnDownNodes, TestAddFieldRealTimeGet, SolrIndexSplitterTes

[...truncated too long message...]

ed
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
   [junit4]   2> 4846526 WARN  (jetty-launcher-4005-thread-2-SendThread(127.0.0.1:48531)) [n:127.0.0.1:49748_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
   [junit4]   2> NOTE: test params are: codec=Lucene70, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@2520c52), locale=es-PA, timezone=Iceland
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=1,free=116147232,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [TestSchemalessBufferedUpdates, LeaderElectionContextKeyTest, PKIAuthenticationIntegrationTest, TestPointFields, BJQParserTest, CoreAdminHandlerTest, TestInPlaceUpdatesStandalone, DocumentBuilderTest, ZkNodePropsTest, TestCloudSearcherWarming, TestJsonFacets, TestNumericTokenStream, TestSimpleTrackingShardHandler, BigEndianAscendingWordDeserializerTest, SecurityConfHandlerTest, TestDistributedMissingSort, TestClusterStateMutator, EchoParamsTest, SuggestComponentTest, TestNonDefinedSimilarityFactory, TestMissingGroups, TestManagedStopFilterFactory, TestGraphMLResponseWriter, TestExpandComponent, TestBadConfig, TestStressVersions, TestSerializedLuceneMatchVersion, DistributedMLTComponentTest, TestObjectReleaseTracker, DebugComponentTest, TestLuceneMatchVersion, TestRecoveryHdfs, TestSolrCoreParser, TestInfoStreamLogging, ConcurrentDeleteAndCreateCollectionTest, AtomicUpdateProcessorFactoryTest, MigrateRouteKeyTest, NumberUtilsTest, SoftAutoCommitTest, PluginInfoTest, TestShardHandlerFactory, TestReplicaProperties, TestDynamicLoading, SimpleFacetsTest, TestDocumentBuilder, TestFastOutputStream, MBeansHandlerTest, SimplePostToolTest, SOLR749Test, TestDeleteCollectionOnDownNodes, TestAddFieldRealTimeGet, SolrIndexSplitterTest, DocValuesMissingTest, FacetPivotSmallTest, TestCloudJSONFacetJoinDomain, DistributedQueryComponentOptimizationTest, TestHashPartitioner, UpdateParamsTest, TestLeaderElectionZkExpiry, CollectionStateFormat2Test, AnalyticsQueryTest, TestMacroExpander, AddBlockUpdateTest, DOMUtilTest, TermVectorComponentTest, CoreAdminCreateDiscoverTest, TestSQLHandler, TestRTGBase, TestLMJelinekMercerSimilarityFactory, QueryElevationComponentTest, MoveReplicaHDFSFailoverTest, TestConfig, DistributedExpandComponentTest, TestCharFilters, TestSolrIndexConfig, BasicFunctionalityTest, UninvertDocValuesMergePolicyTest, TestNamedUpdateProcessors, TestRawTransformer, BitVectorTest, RankQueryTest, TestBlobHandler, TestManagedResourceStorage, UpdateRequestProcessorFactoryTest, TestSort, TestXmlQParserPlugin, TestConfigSetsAPI, CoreMergeIndexesAdminHandlerTest, HdfsRecoveryZkTest, GraphQueryTest, TestLegacyTerms, TestCorePropertiesReload, TestFieldCacheVsDocValues, TestElisionMultitermQuery, CustomCollectionTest, TestSmileRequest, TestMergePolicyConfig, TestSolrCloudSnapshots, JsonLoaderTest, SaslZkACLProviderTest, ShardRoutingCustomTest, LeaderElectionIntegrationTest, ExitableDirectoryReaderTest, RequestLoggingTest, TestSolrCloudWithDelegationTokens, SchemaVersionSpecificBehaviorTest, TestCSVLoader, LeaderFailureAfterFreshStartTest, ManagedSchemaRoundRobinCloudTest, AssignTest, TestSolrQueryParser, CdcrReplicationHandlerTest, V2StandaloneTest, TestHdfsUpdateLog, TestSystemIdResolver, TestSubQueryTransformer, SubstringBytesRefFilterTest, DocValuesMultiTest, LeaderInitiatedRecoveryOnShardRestartTest, HdfsAutoAddReplicasIntegrationTest, TestConfigOverlay, BadComponentTest, TestBulkSchemaAPI, SynonymTokenizerTest, TestPerFieldSimilarity, HdfsSyncSliceTest, PingRequestHandlerTest, SpellCheckCollatorWithCollapseTest, UUIDUpdateProcessorFallbackTest, HdfsNNFailoverTest, QueryResultKeyTest, TestLocalFSCloudBackupRestore, TestRandomFaceting, TestCoreAdminApis, CopyFieldTest, SampleTest, TestFilteredDocIdSet, ReplicaListTransformerTest, TestBM25SimilarityFactory, SharedFSAutoReplicaFailoverTest, TestExtendedDismaxParser, BaseCdcrDistributedZkTest, TestLRUCache, SearchHandlerTest, DistributedFacetPivotLargeTest, TestManagedSynonymGraphFilterFactory, SolrCLIZkUtilsTest, AliasIntegrationTest, TestHdfsCloudBackupRestore, TestUseDocValuesAsStored2, TestFieldTypeCollectionResource, SpellingQueryConverterTest, TestCollationField, HdfsTlogReplayBufferedWhileIndexingTest, CollectionsAPIAsyncDistributedZkTest, IndexSchemaRuntimeFieldTest, TestSearcherReuse, TestSolrJ, CdcrBootstrapTest, ResponseHeaderTest, ZkSolrClientTest, TestSha256AuthenticationProvider, TestReqParamsAPI, TestTolerantUpdateProcessorRandomCloud, QueryEqualityTest, TestWordDelimiterFilterFactory, TestCollectionAPI, TestSystemCollAutoCreate, TestAnalyzedSuggestions, ExternalFileFieldSortTest, BasicAuthIntegrationTest, TestUnifiedSolrHighlighter, TestSchemaSimilarityResource, TestSubQueryTransformerDistrib, TestCloudSchemaless, SolrSlf4jReporterTest, LoggingHandlerTest, StatelessScriptUpdateProcessorFactoryTest, CachingDirectoryFactoryTest, HdfsWriteToMultipleCollectionsTest, TestSearchPerf, TestSchemaNameResource, RollingRestartTest, TestSolrCloudWithSecureImpersonation, TestStressInPlaceUpdates, DeleteStatusTest, BasicDistributedZkTest, LeaderElectionTest, ZkControllerTest, PeerSyncTest, ShowFileRequestHandlerTest, TestCoreDiscovery, TestFoldingMultitermQuery, SuggesterTSTTest, SuggesterTest, TestStressLucene, TestTrie, SpatialFilterTest, PolyFieldTest, NoCacheHeaderTest, WordBreakSolrSpellCheckerTest, TestPseudoReturnFields, TestAtomicUpdateErrorCases, DistanceFunctionTest, CacheHeaderTest, TestSurroundQueryParser, TestQueryUtils, TestWriterPerf, DocumentAnalysisRequestHandlerTest, FileBasedSpellCheckerTest, XmlUpdateRequestHandlerTest, TestValueSourceCache, MoreLikeThisHandlerTest, RequiredFieldsTest, FastVectorHighlighterTest, SolrPluginUtilsTest, QueryParsingTest, BinaryUpdateRequestHandlerTest, TestPHPSerializedResponseWriter, HighlighterConfigTest, ResponseLogComponentTest, TestSolrDeletionPolicy2, MultiTermTest, TestPostingsSolrHighlighter, TestXIncludeConfig, TestFastWriter, OpenExchangeRatesOrgProviderTest, ChaosMonkeyNothingIsSafeTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, ConnectionManagerTest, DeleteInactiveReplicaTest, DeleteLastCustomShardedReplicaTest, DeleteReplicaTest, DistribDocExpirationUpdateProcessorTest, DocValuesNotIndexedTest, ForceLeaderTest, LegacyCloudClusterPropTest, MoveReplicaTest, OverseerModifyCollectionTest, OverseerRolesTest, OverseerStatusTest, OverseerTaskQueueTest, TestAuthenticationFramework, TestCloudDeleteByQuery, TestCloudPseudoReturnFields, TestClusterProperties, TestConfigSetsAPIExclusivity, TestExclusionRuleCollectionAccess, TestOnReconnectListenerSupport, TestPrepRecovery, TestRandomRequestDistribution, TestRebalanceLeaders, TestSegmentSorting, TestStressCloudBlindAtomicUpdates, TestTlogReplica, AutoAddReplicasIntegrationTest, NodeAddedTriggerTest, SystemLogListenerTest, TriggerIntegrationTest, HdfsThreadLeakTest, ZkStateWriterTest, BlobRepositoryCloudTest, TestNRTOpen, TestReloadAndDeleteDocs, TestSimpleTextCodec, TestSolrConfigHandler, TestSolrCoreSnapshots, SearchHandlerTest, TestHdfsBackupRestoreCore, TestSolrConfigHandlerConcurrent, AutoscalingHistoryHandlerTest, JavabinLoaderTest, WrapperMergePolicyFactoryTest, CheckHdfsIndexTest, SolrShardReporterTest, TestManagedResource, TestDynamicFieldCollectionResource, TestUniqueKeyFieldResource, BooleanFieldTest, CurrencyFieldTypeTest, TestBulkSchemaConcurrent, TestManagedSchemaThreadSafety, CursorMarkTest, LargeFieldTest, MergeStrategyTest, TestComplexPhraseLeadingWildcard, TestComplexPhraseQParserPlugin, TestCustomSort, TestGraphTermsQParserPlugin, TestHashQParserPlugin, TestInitQParser, TestLegacyNumericRangeQueryBuilder, TestMultiWordSynonyms, TestNoOpRegenerator, TestOverriddenPrefixQueryForCustomFieldType, TestPayloadCheckQParserPlugin, TestQueryWrapperFilter, TestSolrFieldCacheBean, TestStandardQParsers, TestTrieFacet, TestJsonFacetRefinement, TestMinMaxOnMultiValuedField, TestOrdValues, TestSortByMinMaxFunction, BlockJoinFacetRandomTest, TestScoreJoinQPNoScore, SimpleMLTQParserTest, TestClassicSimilarityFactory, TestDFISimilarityFactory, TestPerFieldSimilarityWithDefaultOverride, TestDefaultStatsCache, TestDistribIDF, TestZkAclsWithHadoopAuth, HttpSolrCallGetCoreTest, HdfsLockFactoryTest, TestDocTermOrds, TestFieldCache, TestFieldCacheReopen, TestFieldCacheSort, TestLegacyFieldCache, TestNumericTerms32, TestNumericTerms64]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestNumericTerms64 -Dtests.seed=A1BAB4070756F514 -Dtests.slow=true -Dtests.locale=es-PA -Dtests.timezone=Iceland -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | TestNumericTerms64 (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: The test or suite printed 13416 bytes to stdout and stderr, even though the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it completely with @SuppressSysoutChecks or run with -Dtests.verbose=true
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A1BAB4070756F514]:0)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4] Completed [720/749 (2!)] on J1 in 6.78s, 4 tests, 1 failure <<< FAILURES!

[...truncated 47331 lines...]