You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/01/24 11:48:07 UTC

[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 631 - Still Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/631/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

2 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:     1) Thread[id=14805, name=jetty-launcher-2141-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)    2) Thread[id=14793, name=jetty-launcher-2141-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   1) Thread[id=14805, name=jetty-launcher-2141-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   2) Thread[id=14793, name=jetty-launcher-2141-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
	at __randomizedtesting.SeedInfo.seed([86D2F4C0C6ED8872]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.uninverting.TestNumericTerms64

Error Message:
The test or suite printed 8944 bytes to stdout and stderr, even though the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it completely with @SuppressSysoutChecks or run with -Dtests.verbose=true

Stack Trace:
java.lang.AssertionError: The test or suite printed 8944 bytes to stdout and stderr, even though the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it completely with @SuppressSysoutChecks or run with -Dtests.verbose=true
	at __randomizedtesting.SeedInfo.seed([86D2F4C0C6ED8872]:0)
	at org.apache.lucene.util.TestRuleLimitSysouts.afterIfSuccessful(TestRuleLimitSysouts.java:211)
	at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterIfSuccessful(TestRuleAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:37)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11567 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_86D2F4C0C6ED8872-001/init-core-data-001
   [junit4]   2> 1161080 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[86D2F4C0C6ED8872]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1161113 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[86D2F4C0C6ED8872]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_86D2F4C0C6ED8872-001/tempDir-001
   [junit4]   2> 1161114 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[86D2F4C0C6ED8872]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1161114 INFO  (Thread-5435) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1161114 INFO  (Thread-5435) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1161214 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[86D2F4C0C6ED8872]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:59804
   [junit4]   2> 1161225 INFO  (jetty-launcher-2141-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1161225 INFO  (jetty-launcher-2141-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1161230 INFO  (jetty-launcher-2141-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5826b46{/solr,null,AVAILABLE}
   [junit4]   2> 1161230 INFO  (jetty-launcher-2141-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6347523a{/solr,null,AVAILABLE}
   [junit4]   2> 1161230 INFO  (jetty-launcher-2141-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@a7f7fc4{HTTP/1.1,[http/1.1]}{127.0.0.1:38860}
   [junit4]   2> 1161230 INFO  (jetty-launcher-2141-thread-2) [    ] o.e.j.s.Server Started @1165141ms
   [junit4]   2> 1161230 INFO  (jetty-launcher-2141-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38860}
   [junit4]   2> 1161231 INFO  (jetty-launcher-2141-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7b1b8ff4{HTTP/1.1,[http/1.1]}{127.0.0.1:35766}
   [junit4]   2> 1161231 INFO  (jetty-launcher-2141-thread-1) [    ] o.e.j.s.Server Started @1165141ms
   [junit4]   2> 1161231 ERROR (jetty-launcher-2141-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1161231 INFO  (jetty-launcher-2141-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35766}
   [junit4]   2> 1161231 INFO  (jetty-launcher-2141-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 1161231 ERROR (jetty-launcher-2141-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1161231 INFO  (jetty-launcher-2141-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1161231 INFO  (jetty-launcher-2141-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1161231 INFO  (jetty-launcher-2141-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 1161231 INFO  (jetty-launcher-2141-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-24T10:50:49.319Z
   [junit4]   2> 1161231 INFO  (jetty-launcher-2141-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1161231 INFO  (jetty-launcher-2141-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1161231 INFO  (jetty-launcher-2141-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-24T10:50:49.319Z
   [junit4]   2> 1161236 INFO  (jetty-launcher-2141-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1161236 INFO  (jetty-launcher-2141-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1161247 INFO  (jetty-launcher-2141-thread-1) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1161248 INFO  (jetty-launcher-2141-thread-2) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1161249 INFO  (jetty-launcher-2141-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59804/solr
   [junit4]   2> 1161252 INFO  (jetty-launcher-2141-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59804/solr
   [junit4]   2> 1161321 INFO  (jetty-launcher-2141-thread-1) [n:127.0.0.1:35766_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:35766_solr
   [junit4]   2> 1161323 INFO  (jetty-launcher-2141-thread-1) [n:127.0.0.1:35766_solr    ] o.a.s.c.Overseer Overseer (id=97337674904633349-127.0.0.1:35766_solr-n_0000000000) starting
   [junit4]   2> 1161329 INFO  (jetty-launcher-2141-thread-2) [n:127.0.0.1:38860_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38860_solr
   [junit4]   2> 1161333 INFO  (zkCallback-2151-thread-1-processing-n:127.0.0.1:35766_solr) [n:127.0.0.1:35766_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1161334 INFO  (zkCallback-2152-thread-1-processing-n:127.0.0.1:38860_solr) [n:127.0.0.1:38860_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1161346 INFO  (jetty-launcher-2141-thread-1) [n:127.0.0.1:35766_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35766_solr
   [junit4]   2> 1161349 INFO  (zkCallback-2151-thread-1-processing-n:127.0.0.1:35766_solr) [n:127.0.0.1:35766_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1161349 INFO  (zkCallback-2152-thread-1-processing-n:127.0.0.1:38860_solr) [n:127.0.0.1:38860_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1161399 INFO  (jetty-launcher-2141-thread-1) [n:127.0.0.1:35766_solr    ] o.a.s.c.CoreContainer Authentication plugin class obtained from system property 'authenticationPlugin': org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 1161399 INFO  (jetty-launcher-2141-thread-1) [n:127.0.0.1:35766_solr    ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=solaris-vm,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
   [junit4]   2> 1161400 INFO  (jetty-launcher-2141-thread-1) [n:127.0.0.1:35766_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 1161405 INFO  (jetty-launcher-2141-thread-1-EventThread) [n:127.0.0.1:35766_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 1161435 INFO  (jetty-launcher-2141-thread-1) [n:127.0.0.1:35766_solr    ] o.a.s.h.c.HttpShardHandlerFactory Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@2b6ea225
   [junit4]   2> 1161436 INFO  (jetty-launcher-2141-thread-1) [n:127.0.0.1:35766_solr    ] o.a.s.u.UpdateShardHandler Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@2b6ea225
   [junit4]   2> 1161440 INFO  (jetty-launcher-2141-thread-1) [n:127.0.0.1:35766_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_86D2F4C0C6ED8872-001/tempDir-001/node1/.
   [junit4]   2> 1161571 INFO  (jetty-launcher-2141-thread-2) [n:127.0.0.1:38860_solr    ] o.a.s.c.CoreContainer Authentication plugin class obtained from system property 'authenticationPlugin': org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 1161571 INFO  (jetty-launcher-2141-thread-2) [n:127.0.0.1:38860_solr    ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=solaris-vm,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
   [junit4]   2> 1161572 INFO  (jetty-launcher-2141-thread-2) [n:127.0.0.1:38860_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 1161576 INFO  (jetty-launcher-2141-thread-2-EventThread) [n:127.0.0.1:38860_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 1161600 INFO  (jetty-launcher-2141-thread-2) [n:127.0.0.1:38860_solr    ] o.a.s.h.c.HttpShardHandlerFactory Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@2fd42114
   [junit4]   2> 1161600 INFO  (jetty-launcher-2141-thread-2) [n:127.0.0.1:38860_solr    ] o.a.s.u.UpdateShardHandler Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@2fd42114
   [junit4]   2> 1161603 INFO  (jetty-launcher-2141-thread-2) [n:127.0.0.1:38860_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_86D2F4C0C6ED8872-001/tempDir-001/node2/.
   [junit4]   2> 1162206 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateHost
   [junit4]   2> 1162216 INFO  (qtp1023582788-14753) [n:127.0.0.1:35766_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1162216 INFO  (qtp1023582788-14753) [n:127.0.0.1:35766_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1162217 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[86D2F4C0C6ED8872]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=test&e=1485291050298&s=A/BOBZ8zmY1O5B/5KBNh18HSW3E="; Path=/; Domain=127.0.0.1; Expires=mar, 24-ene-2017 20:50:50 GMT; HttpOnly". Invalid 'expires' attribute: mar, 24-ene-2017 20:50:50 GMT
   [junit4]   2> 1162218 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateHost
   [junit4]   2> 1162244 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNullProxyUser
   [junit4]   2> 1162246 INFO  (qtp1023582788-14755) [    ] o.a.s.s.SolrDispatchFilter Error authenticating
   [junit4]   2> java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:220)
   [junit4]   2> 	at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> 	at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:519)
   [junit4]   2> 	at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:128)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:252)
   [junit4]   2> 	at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:248)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:404)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:310)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:296)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:110)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1162247 WARN  (qtp1023582788-14755) [    ] o.e.j.s.ServletHandler /solr/admin/collections
   [junit4]   2> org.apache.solr.common.SolrException: Error during request authentication, 
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:410)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:310)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:296)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:110)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:220)
   [junit4]   2> 	at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> 	at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:519)
   [junit4]   2> 	at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:128)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:252)
   [junit4]   2> 	at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:248)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:404)
   [junit4]   2> 	... 26 more
   [junit4]   2> 1162248 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNullProxyUser
   [junit4]   2> 1162276 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidProxyUser
   [junit4]   2> 1162281 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[86D2F4C0C6ED8872]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bar&p=bar&t=test&e=1485291050366&s=iJiIHjwlNg8PGqZlAKyN2T4gRjs="; Path=/; Domain=127.0.0.1; Expires=mar, 24-ene-2017 20:50:50 GMT; HttpOnly". Invalid 'expires' attribute: mar, 24-ene-2017 20:50:50 GMT
   [junit4]   2> 1162281 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidProxyUser
   [junit4]   2> 1162316 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidGroup
   [junit4]   2> 1162340 WARN  (qtp1023582788-14762) [    ] o.a.h.s.ShellBasedUnixGroupsMapping got exception trying to get groups for user bar: id: invalid user name: "bar"
   [junit4]   2> 
   [junit4]   2> 1162341 WARN  (qtp1023582788-14762) [    ] o.a.h.s.UserGroupInformation No groups available for user bar
   [junit4]   2> 1162343 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[86D2F4C0C6ED8872]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bogusGroup&p=bogusGroup&t=test&e=1485291050407&s=JsasX/KIHq4oZEM4hj0J0kLpcL0="; Path=/; Domain=127.0.0.1; Expires=mar, 24-ene-2017 20:50:50 GMT; HttpOnly". Invalid 'expires' attribute: mar, 24-ene-2017 20:50:50 GMT
   [junit4]   2> 1162343 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidGroup
   [junit4]   2> 1162372 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateGroup
   [junit4]   2> 1162376 INFO  (qtp1023582788-14747) [n:127.0.0.1:35766_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=jenkins&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1162376 INFO  (qtp1023582788-14747) [n:127.0.0.1:35766_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=jenkins&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1162376 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[86D2F4C0C6ED8872]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostUsersGroup&p=anyHostUsersGroup&t=test&e=1485291050463&s=QgUKRUCwkLbg4SWD335JWZIINXw="; Path=/; Domain=127.0.0.1; Expires=mar, 24-ene-2017 20:50:50 GMT; HttpOnly". Invalid 'expires' attribute: mar, 24-ene-2017 20:50:50 GMT
   [junit4]   2> 1162376 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateGroup
   [junit4]   2> 1162407 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testForwarding
   [junit4]   2> 1163269 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1163270 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59804/solr ready
   [junit4]   2> 1163275 INFO  (qtp1996861728-14749) [n:127.0.0.1:38860_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1163280 INFO  (OverseerThreadFactory-5133-thread-1-processing-n:127.0.0.1:35766_solr) [n:127.0.0.1:35766_solr    ] o.a.s.c.CreateCollectionCmd Create collection forwardingCollection
   [junit4]   2> 1163393 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1163506 INFO  (zkCallback-2152-thread-1-processing-n:127.0.0.1:38860_solr) [n:127.0.0.1:38860_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1164422 WARN  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1164424 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 1164457 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.IndexSchema [forwardingCollection_shard1_replica1] Schema name=test
   [junit4]   2> 1164541 WARN  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.IndexSchema [forwardingCollection_shard1_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1164542 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1164564 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'forwardingCollection_shard1_replica1' using configuration from collection forwardingCollection
   [junit4]   2> 1164565 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SolrCore [[forwardingCollection_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_86D2F4C0C6ED8872-001/tempDir-001/node2/forwardingCollection_shard1_replica1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_86D2F4C0C6ED8872-001/tempDir-001/node2/./forwardingCollection_shard1_replica1/data/]
   [junit4]   2> 1164565 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f06b730
   [junit4]   2> 1164567 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=38, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6629629392881913]
   [junit4]   2> 1164592 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1164592 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1164593 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1164593 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1164594 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.38533440192454005]
   [junit4]   2> 1164594 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@31d87ebf[forwardingCollection_shard1_replica1] main]
   [junit4]   2> 1164596 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 1164600 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 1164628 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1164629 INFO  (searcherExecutor-5138-thread-1-processing-n:127.0.0.1:38860_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 1164629 INFO  (searcherExecutor-5138-thread-1-processing-n:127.0.0.1:38860_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 1164630 INFO  (searcherExecutor-5138-thread-1-processing-n:127.0.0.1:38860_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 1164630 INFO  (searcherExecutor-5138-thread-1-processing-n:127.0.0.1:38860_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 1164630 INFO  (searcherExecutor-5138-thread-1-processing-n:127.0.0.1:38860_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 1164630 INFO  (searcherExecutor-5138-thread-1-processing-n:127.0.0.1:38860_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 1164631 INFO  (searcherExecutor-5138-thread-1-processing-n:127.0.0.1:38860_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 1164631 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557402802159878144
   [junit4]   2> 1164631 INFO  (searcherExecutor-5138-thread-1-processing-n:127.0.0.1:38860_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 1164632 INFO  (searcherExecutor-5138-thread-1-processing-n:127.0.0.1:38860_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 1164633 INFO  (searcherExecutor-5138-thread-1-processing-n:127.0.0.1:38860_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica1] Registered new searcher Searcher@31d87ebf[forwardingCollection_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1164644 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1164644 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1164644 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:38860/solr/forwardingCollection_shard1_replica1/
   [junit4]   2> 1164644 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1164644 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:38860/solr/forwardingCollection_shard1_replica1/ has no replicas
   [junit4]   2> 1164651 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:38860/solr/forwardingCollection_shard1_replica1/ shard1
   [junit4]   2> 1164756 INFO  (zkCallback-2152-thread-1-processing-n:127.0.0.1:38860_solr) [n:127.0.0.1:38860_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1164803 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1164807 INFO  (qtp1996861728-14752) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2} status=0 QTime=1414
   [junit4]   2> 1164807 WARN  (httpShardExecutor-5131-thread-1-processing-n:127.0.0.1:35766_solr [http:////127.0.0.1:38860//solr] http:////127.0.0.1:38860//solr) [n:127.0.0.1:35766_solr    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=$&p=$&t=test&e=1485291051480&s=GV0+T+yCWI+KmboR0AC2hi4rhb8="; Path=/; Domain=127.0.0.1; Expires=mar, 24-ene-2017 20:50:51 GMT; HttpOnly". Invalid 'expires' attribute: mar, 24-ene-2017 20:50:51 GMT
   [junit4]   2> 1164812 INFO  (qtp1996861728-14749) [n:127.0.0.1:38860_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1164911 INFO  (zkCallback-2152-thread-1-processing-n:127.0.0.1:38860_solr) [n:127.0.0.1:38860_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1165285 INFO  (OverseerCollectionConfigSetProcessor-97337674904633349-127.0.0.1:35766_solr-n_0000000000) [n:127.0.0.1:35766_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1165813 INFO  (qtp1996861728-14749) [n:127.0.0.1:38860_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2538
   [junit4]   2> 1165814 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[86D2F4C0C6ED8872]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=user&p=user&t=test&e=1485291051362&s=YOAkuflawXmgFf+sd737WXqshkk="; Path=/; Domain=127.0.0.1; Expires=mar, 24-ene-2017 20:50:51 GMT; HttpOnly". Invalid 'expires' attribute: mar, 24-ene-2017 20:50:51 GMT
   [junit4]   2> 1165814 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: forwardingCollection failOnTimeout:true timeout (sec):100
   [junit4]   2> 1165815 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: forwardingCollection
   [junit4]   2> 1165824 INFO  (qtp1996861728-14756) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.c.S.Request [forwardingCollection_shard1_replica1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 1165825 WARN  (qtp1023582788-14755) [n:127.0.0.1:35766_solr    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=$&p=$&t=test&e=1485291053911&s=pHGa1yOVFUhRI7tmRfwUW60Qq3U="; Path=/; Domain=127.0.0.1; Expires=mar, 24-ene-2017 20:50:53 GMT; HttpOnly". Invalid 'expires' attribute: mar, 24-ene-2017 20:50:53 GMT
   [junit4]   2> 1165830 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[86D2F4C0C6ED8872]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=user&p=user&t=test&e=1485291053905&s=xxn7F0TaebatVdi1fCXD/G43g6Q="; Path=/; Domain=127.0.0.1; Expires=mar, 24-ene-2017 20:50:53 GMT; HttpOnly". Invalid 'expires' attribute: mar, 24-ene-2017 20:50:53 GMT
   [junit4]   2> 1165830 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[86D2F4C0C6ED8872]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=$&p=$&t=test&e=1485291053911&s=pHGa1yOVFUhRI7tmRfwUW60Qq3U="; Path=/; Domain=127.0.0.1; Expires=mar, 24-ene-2017 20:50:53 GMT; HttpOnly". Invalid 'expires' attribute: mar, 24-ene-2017 20:50:53 GMT
   [junit4]   2> 1165834 INFO  (qtp1996861728-14746) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.c.S.Request [forwardingCollection_shard1_replica1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 1165834 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[86D2F4C0C6ED8872]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=user&p=user&t=test&e=1485291053921&s=5fx/++tJCG1MUppU1vOdzonSaAQ="; Path=/; Domain=127.0.0.1; Expires=mar, 24-ene-2017 20:50:53 GMT; HttpOnly". Invalid 'expires' attribute: mar, 24-ene-2017 20:50:53 GMT
   [junit4]   2> 1165835 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testForwarding
   [junit4]   2> 1165879 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigHosts
   [junit4]   2> 1165885 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[86D2F4C0C6ED8872]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noHosts&p=noHosts&t=test&e=1485291053970&s=FAmr/CBhWeCZSgH4aLBATZ9e92E="; Path=/; Domain=127.0.0.1; Expires=mar, 24-ene-2017 20:50:53 GMT; HttpOnly". Invalid 'expires' attribute: mar, 24-ene-2017 20:50:53 GMT
   [junit4]   2> 1165885 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigHosts
   [junit4]   2> 1165922 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateAnyHostAnyUser
   [junit4]   2> 1165924 INFO  (qtp1023582788-14762) [n:127.0.0.1:35766_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1165925 INFO  (qtp1023582788-14762) [n:127.0.0.1:35766_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1165925 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[86D2F4C0C6ED8872]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostAnyUser&p=anyHostAnyUser&t=test&e=1485291054011&s=kKNCJm3MzFG1dWPXjdogzQQqsi4="; Path=/; Domain=127.0.0.1; Expires=mar, 24-ene-2017 20:50:54 GMT; HttpOnly". Invalid 'expires' attribute: mar, 24-ene-2017 20:50:54 GMT
   [junit4]   2> 1165925 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateAnyHostAnyUser
   [junit4]   2> 1165954 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigGroups
   [junit4]   2> 1165959 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[86D2F4C0C6ED8872]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noGroups&p=noGroups&t=test&e=1485291054044&s=5MLs/3O2X2zdhr1dnHjGDc0CHMs="; Path=/; Domain=127.0.0.1; Expires=mar, 24-ene-2017 20:50:54 GMT; HttpOnly". Invalid 'expires' attribute: mar, 24-ene-2017 20:50:54 GMT
   [junit4]   2> 1165959 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigGroups
   [junit4]   2> 1166000 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyWrongHost
   [junit4]   2> 1166020 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[86D2F4C0C6ED8872]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=wrongHost&p=wrongHost&t=test&e=1485291054093&s=jS4fxV/tIoKmVrLKYAZxseOPi/Q="; Path=/; Domain=127.0.0.1; Expires=mar, 24-ene-2017 20:50:54 GMT; HttpOnly". Invalid 'expires' attribute: mar, 24-ene-2017 20:50:54 GMT
   [junit4]   2> 1166021 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyWrongHost
   [junit4]   2> 1166053 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidRemote
   [junit4]   2> 1166062 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[86D2F4C0C6ED8872]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=test&e=1485291054143&s=3jfnl4M7TfwnAJKzFyfM+LXPcDU="; Path=/; Domain=127.0.0.1; Expires=mar, 24-ene-2017 20:50:54 GMT; HttpOnly". Invalid 'expires' attribute: mar, 24-ene-2017 20:50:54 GMT
   [junit4]   2> 1166062 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidRemote
   [junit4]   2> 1166095 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyUnknownRemote
   [junit4]   2> 1181462 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159d0197ff40008, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1181462 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159d0197ff40006, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1181462 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159d0197ff40007, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1181463 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x159d0197ff40005 due to java.io.IOException: Connection reset by peer
   [junit4]   2> 1181463 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[86D2F4C0C6ED8872]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=test&e=1485291054193&s=MM6qj1tjCwoq2cbqNhiy82HgFyo="; Path=/; Domain=127.0.0.1; Expires=mar, 24-ene-2017 20:50:54 GMT; HttpOnly". Invalid 'expires' attribute: mar, 24-ene-2017 20:50:54 GMT
   [junit4]   2> 1181467 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[86D2F4C0C6ED8872]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyUnknownRemote
   [junit4]   2> 1181562 INFO  (jetty-launcher-2141-thread-1-EventThread) [n:127.0.0.1:35766_solr    ] o.a.c.f.s.ConnectionStateManager State change: SUSPENDED
   [junit4]   2> 1181565 WARN  (zkCallback-2152-thread-1-processing-n:127.0.0.1:38860_solr) [n:127.0.0.1:38860_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@647b5d59 name: ZooKeeperConnection Watcher:127.0.0.1:59804/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1181565 WARN  (zkCallback-2152-thread-1-processing-n:127.0.0.1:38860_solr) [n:127.0.0.1:38860_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 1181571 WARN  (zkCallback-2151-thread-2-processing-n:127.0.0.1:35766_solr) [n:127.0.0.1:35766_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@63b08c98 name: ZooKeeperConnection Watcher:127.0.0.1:59804/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1181572 WARN  (zkCallback-2151-thread-2-processing-n:127.0.0.1:35766_solr) [n:127.0.0.1:35766_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 1181587 INFO  (jetty-launcher-2141-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@7b1b8ff4{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1181589 INFO  (jetty-launcher-2141-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1282684261
   [junit4]   2> 1181589 INFO  (jetty-launcher-2141-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@a7f7fc4{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1181589 INFO  (jetty-launcher-2141-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=337604238
   [junit4]   2> 1183085 INFO  (coreCloseExecutor-5143-thread-1) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@3177506a
   [junit4]   2> 1183126 INFO  (coreCloseExecutor-5143-thread-1) [n:127.0.0.1:38860_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.forwardingCollection.shard1.replica1
   [junit4]   2> 1183128 WARN  (zkCallback-2152-thread-1-processing-n:127.0.0.1:38860_solr) [n:127.0.0.1:38860_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1183129 INFO  (jetty-launcher-2141-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1183130 ERROR (Thread[Thread-5455,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:38860_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 1183136 INFO  (jetty-launcher-2141-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5826b46{/solr,null,UNAVAILABLE}
   [junit4]   2> 1183262 INFO  (zkCallback-2151-thread-4-processing-n:127.0.0.1:35766_solr) [n:127.0.0.1:35766_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 1183266 INFO  (jetty-launcher-2141-thread-2) [    ] o.a.s.c.Overseer Overseer (id=97337674904633349-127.0.0.1:35766_solr-n_0000000000) closing
   [junit4]   2> 1183266 INFO  (OverseerStateUpdate-97337674904633349-127.0.0.1:35766_solr-n_0000000000) [n:127.0.0.1:35766_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:35766_solr
   [junit4]   2> 1183268 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159d0197ff40005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1184769 WARN  (zkCallback-2151-thread-4-processing-n:127.0.0.1:35766_solr) [n:127.0.0.1:35766_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1184770 INFO  (jetty-launcher-2141-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1184770 ERROR (Thread[Thread-5452,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:35766_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 1184773 INFO  (jetty-launcher-2141-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6347523a{/solr,null,UNAVAILABLE}
   [junit4]   2> 1184780 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[86D2F4C0C6ED8872]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:59804 59804
   [junit4]   2> 1184839 INFO  (Thread-5435) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:59804 59804
   [junit4]   2> 1184841 WARN  (Thread-5435) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/overseer/queue
   [junit4]   2> 	6	/solr/overseer/collection-queue-work
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	3	/solr/collections
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 
   [junit4]   2> 1184924 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[86D2F4C0C6ED8872]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> Jan 24, 2017 10:51:13 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> Jan 24, 2017 10:51:33 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   [junit4]   2>    1) Thread[id=14805, name=jetty-launcher-2141-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2>         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2>         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2>         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2>         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2>    2) Thread[id=14793, name=jetty-launcher-2141-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2>         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2>         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2>         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2>         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> Jan 24, 2017 10:51:33 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=14805, name=jetty-launcher-2141-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2>    2) Thread[id=14793, name=jetty-launcher-2141-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2> 1205109 ERROR (jetty-launcher-2141-thread-1-EventThread) [n:127.0.0.1:35766_solr    ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2> 	at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2> 	at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2> 	at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2> 	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 1205109 ERROR (jetty-launcher-2141-thread-2-EventThread) [n:127.0.0.1:38860_solr    ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2> 	at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2> 	at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2> 	at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2> 	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 1205111 WARN  (jetty-launcher-2141-thread-1-EventThread) [n:127.0.0.1:35766_solr    ] o.a.c.ConnectionState Session expired event received
   [junit4]   2> 1205111 WARN  (jetty-launcher-2141-thread-2-EventThread) [n:127.0.0.1:38860_solr    ] o.a.c.ConnectionState Session expired event received
   [junit4]   2> 1205112 WARN  (jetty-launcher-2141-thread-1-SendThread(127.0.0.1:59804)) [n:127.0.0.1:35766_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   [junit4]   2> Jan 24, 2017 10:51:33 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> 1205112 WARN  (jetty-launcher-2141-thread-2-SendThread(127.0.0.1:59804)) [n:127.0.0.1:38860_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true,coord=no): {}, locale=es-NI, timezone=America/Argentina/Tucuman
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=5,free=207945392,total=501219328
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrQueryParserResource, TestCoreContainer, TestSurroundQueryParser, TestHdfsCloudBackupRestore, SolrMetricReporterTest, CdcrRequestHandlerTest, PeerSyncReplicationTest, ManagedSchemaRoundRobinCloudTest, TestFieldCollectionResource, TestHighlightDedupGrouping, ZkControllerTest, SolrCoreTest, TestLFUCache, SolrCloudExampleTest, TestStressRecovery, SolrCoreMetricManagerTest, TestNoOpRegenerator, RequestLoggingTest, TestMaxScoreQueryParser, CollectionStateFormat2Test, TestWordDelimiterFilterFactory, TestSubQueryTransformer, DocumentBuilderTest, TestFunctionQuery, ConnectionManagerTest, TestPivotHelperCode, TestFreeTextSuggestions, TestAuthenticationFramework, AlternateDirectoryTest, TestPostingsSolrHighlighter, TestCorePropertiesReload, TestTrieFacet, DocValuesMissingTest, CdcrReplicationDistributedZkTest, TestRecoveryHdfs, LeaderElectionTest, BitVectorTest, ZkNodePropsTest, MultiTermTest, PathHierarchyTokenizerFactoryTest, TestStressLucene, AtomicUpdatesTest, BasicDistributedZkTest, JavabinLoaderTest, TestShortCircuitedRequests, HdfsCollectionsAPIDistributedZkTest, TestStressLiveNodes, SharedFSAutoReplicaFailoverTest, CustomCollectionTest, TestSubQueryTransformerDistrib, SortSpecParsingTest, TestDynamicFieldCollectionResource, HdfsThreadLeakTest, SynonymTokenizerTest, TestSmileRequest, TestMiniSolrCloudClusterSSL, TestFaceting, JSONWriterTest, SuggestComponentContextFilterQueryTest, TestQueryWrapperFilter, DOMUtilTest, TestCursorMarkWithoutUniqueKey, SmileWriterTest, TestOrdValues, DistributedFacetPivotWhiteBoxTest, TestUnifiedSolrHighlighter, TestManagedStopFilterFactory, TestSSLRandomization, SyncSliceTest, QueryResultKeyTest, TestScoreJoinQPScore, MultiThreadedOCPTest, XsltUpdateRequestHandlerTest, TestSchemaNameResource, TestReplicationHandler, ClassificationUpdateProcessorIntegrationTest, TestNonDefinedSimilarityFactory, DistributedFacetPivotSmallAdvancedTest, TestFastOutputStream, TestBinaryField, SmileWriterTest, TestRebalanceLeaders, NumericFieldsTest, TestSort, SecurityConfHandlerTest, PKIAuthenticationIntegrationTest, MigrateRouteKeyTest, SimplePostToolTest, PeerSyncTest, TestDynamicLoading, TestManagedSchemaThreadSafety, TriLevelCompositeIdRoutingTest, TestExtendedDismaxParser, IndexSchemaRuntimeFieldTest, CloneFieldUpdateProcessorFactoryTest, TestUninvertingReader, TestSolrDynamicMBean, FullHLLTest, TestXmlQParserPlugin, ResponseLogComponentTest, TestSearcherReuse, PrimitiveFieldTypeTest, TestSolrCloudWithDelegationTokens, BigEndianAscendingWordDeserializerTest, DistributedQueryComponentOptimizationTest, XmlUpdateRequestHandlerTest, TestPKIAuthenticationPlugin, TemplateUpdateProcessorTest, TestHashPartitioner, BadCopyFieldTest, OverriddenZkACLAndCredentialsProvidersTest, TestIBSimilarityFactory, TestBlendedInfixSuggestions, TestStressVersions, DirectUpdateHandlerTest, TermsComponentTest, TestJmxIntegration, FacetPivotSmallTest, HdfsLockFactoryTest, TestCSVLoader, TestRTimerTree, BlockJoinFacetRandomTest, TestFieldCacheWithThreads, TestManagedSchema, TestConfigSetsAPI, TestSolrCloudWithSecureImpersonation]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=86D2F4C0C6ED8872 -Dtests.slow=true -Dtests.locale=es-NI -Dtests.timezone=America/Argentina/Tucuman -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J0 | TestSolrCloudWithSecureImpersonation (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   [junit4]    >    1) Thread[id=14805, name=jetty-launcher-2141-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]    >    2) Thread[id=14793, name=jetty-launcher-2141-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([86D2F4C0C6ED8872]:0)
   [junit4] Completed [253/681 (1!)] on J0 in 44.05s, 12 tests, 1 error <<< FAILURES!

[...truncated 1268 lines...]
   [junit4] Suite: org.apache.solr.uninverting.TestNumericTerms64
   [junit4]   2> 3139748 WARN  (jetty-launcher-2141-thread-1-SendThread(127.0.0.1:59804)) [n:127.0.0.1:35766_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   [junit4]   2> 3139752 WARN  (jetty-launcher-2141-thread-2-SendThread(127.0.0.1:59804)) [n:127.0.0.1:38860_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   [junit4]   2> 3140851 WARN  (jetty-launcher-2141-thread-1-SendThread(127.0.0.1:59804)) [n:127.0.0.1:35766_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   [junit4]   2> 3140853 WARN  (jetty-launcher-2141-thread-2-SendThread(127.0.0.1:59804)) [n:127.0.0.1:38860_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   [junit4]   2> 3141953 WARN  (jetty-launcher-2141-thread-1-SendThread(127.0.0.1:59804)) [n:127.0.0.1:35766_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   [junit4]   2> 3141954 WARN  (jetty-launcher-2141-thread-2-SendThread(127.0.0.1:59804)) [n:127.0.0.1:38860_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   [junit4]   2> 3143058 WARN  (jetty-launcher-2141-thread-1-SendThread(127.0.0.1:59804)) [n:127.0.0.1:35766_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   [junit4]   2> 3143059 WARN  (jetty-launcher-2141-thread-2-SendThread(127.0.0.1:59804)) [n:127.0.0.1:38860_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62), sim=RandomSimilarity(queryNorm=false,coord=yes): {}, locale=ar-YE, timezone=Etc/Greenwich
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=113192664,total=505413632
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrQueryParserResource, TestCoreContainer, TestSurroundQueryParser, TestHdfsCloudBackupRestore, SolrMetricReporterTest, CdcrRequestHandlerTest, PeerSyncReplicationTest, ManagedSchemaRoundRobinCloudTest, TestFieldCollectionResource, TestHighlightDedupGrouping, ZkControllerTest, SolrCoreTest, TestLFUCache, SolrCloudExampleTest, TestStressRecovery, SolrCoreMetricManagerTest, TestNoOpRegenerator, RequestLoggingTest, TestMaxScoreQueryParser, CollectionStateFormat2Test, TestWordDelimiterFilterFactory, TestSubQueryTransformer, DocumentBuilderTest, TestFunctionQuery, ConnectionManagerTest, TestPivotHelperCode, TestFreeTextSuggestions, TestAuthenticationFramework, AlternateDirectoryTest, TestPostingsSolrHighlighter, TestCorePropertiesReload, TestTrieFacet, DocValuesMissingTest, CdcrReplicationDistributedZkTest, TestRecoveryHdfs, LeaderElectionTest, BitVectorTest, ZkNodePropsTest, MultiTermTest, PathHierarchyTokenizerFactoryTest, TestStressLucene, AtomicUpdatesTest, BasicDistributedZkTest, JavabinLoaderTest, TestShortCircuitedRequests, HdfsCollectionsAPIDistributedZkTest, TestStressLiveNodes, SharedFSAutoReplicaFailoverTest, CustomCollectionTest, TestSubQueryTransformerDistrib, SortSpecParsingTest, TestDynamicFieldCollectionResource, HdfsThreadLeakTest, SynonymTokenizerTest, TestSmileRequest, TestMiniSolrCloudClusterSSL, TestFaceting, JSONWriterTest, SuggestComponentContextFilterQueryTest, TestQueryWrapperFilter, DOMUtilTest, TestCursorMarkWithoutUniqueKey, SmileWriterTest, TestOrdValues, DistributedFacetPivotWhiteBoxTest, TestUnifiedSolrHighlighter, TestManagedStopFilterFactory, TestSSLRandomization, SyncSliceTest, QueryResultKeyTest, TestScoreJoinQPScore, MultiThreadedOCPTest, XsltUpdateRequestHandlerTest, TestSchemaNameResource, TestReplicationHandler, ClassificationUpdateProcessorIntegrationTest, TestNonDefinedSimilarityFactory, DistributedFacetPivotSmallAdvancedTest, TestFastOutputStream, TestBinaryField, SmileWriterTest, TestRebalanceLeaders, NumericFieldsTest, TestSort, SecurityConfHandlerTest, PKIAuthenticationIntegrationTest, MigrateRouteKeyTest, SimplePostToolTest, PeerSyncTest, TestDynamicLoading, TestManagedSchemaThreadSafety, TriLevelCompositeIdRoutingTest, TestExtendedDismaxParser, IndexSchemaRuntimeFieldTest, CloneFieldUpdateProcessorFactoryTest, TestUninvertingReader, TestSolrDynamicMBean, FullHLLTest, TestXmlQParserPlugin, ResponseLogComponentTest, TestSearcherReuse, PrimitiveFieldTypeTest, TestSolrCloudWithDelegationTokens, BigEndianAscendingWordDeserializerTest, DistributedQueryComponentOptimizationTest, XmlUpdateRequestHandlerTest, TestPKIAuthenticationPlugin, TemplateUpdateProcessorTest, TestHashPartitioner, BadCopyFieldTest, OverriddenZkACLAndCredentialsProvidersTest, TestIBSimilarityFactory, TestBlendedInfixSuggestions, TestStressVersions, DirectUpdateHandlerTest, TermsComponentTest, TestJmxIntegration, FacetPivotSmallTest, HdfsLockFactoryTest, TestCSVLoader, TestRTimerTree, BlockJoinFacetRandomTest, TestFieldCacheWithThreads, TestManagedSchema, TestConfigSetsAPI, TestSolrCloudWithSecureImpersonation, TestCustomSort, TestSweetSpotSimilarityFactory, MoreLikeThisHandlerTest, FastVectorHighlighterTest, TestCollationField, TestSolrIndexConfig, TestLocalFSCloudBackupRestore, TestJsonFacets, TestConfigSetProperties, IgnoreCommitOptimizeUpdateProcessorFactoryTest, CloudExitableDirectoryReaderTest, TestManagedResourceStorage, CollectionsAPIAsyncDistributedZkTest, TestTestInjection, TestFieldTypeCollectionResource, TestUpdate, TestCharFilters, TestFastLRUCache, CoreAdminRequestStatusTest, TestFieldCache, TestSerializedLuceneMatchVersion, TestJettySolrRunner, TestElisionMultitermQuery, QueryEqualityTest, TestNRTOpen, TestSQLHandlerNonCloud, TestImpersonationWithHadoopAuth, IndexBasedSpellCheckerTest, TestDownShardTolerantSearch, PingRequestHandlerTest, TestExactSharedStatsCache, TestClassicSimilarityFactory, MetricUtilsTest, TestRandomFlRTGCloud, DocValuesTest, HttpPartitionTest, BJQParserTest, SpatialRPTFieldTypeTest, DirectSolrConnectionTest, TestInfoStreamLogging, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, OpenCloseCoreStressTest, OverseerTest, LeaderElectionIntegrationTest, BasicZkTest, RecoveryZkTest, ClusterStateUpdateTest, TestZkChroot, ZkCLITest, AutoCommitTest, TestRecovery, DistributedSpellCheckComponentTest, TestStressReorder, TestJoin, HardAutoCommitTest, DistributedTermsComponentTest, SimpleFacetsTest, TestGroupingSearch, QueryElevationComponentTest, BadIndexSchemaTest, TestFiltering, TestLazyCores, HighlighterTest, SoftAutoCommitTest, CurrencyFieldOpenExchangeTest, DistributedQueryElevationComponentTest, CurrencyFieldXmlFileTest, SolrIndexSplitterTest, AnalysisAfterCoreReloadTest, SignatureUpdateProcessorFactoryTest, SpellCheckCollatorTest, SuggesterFSTTest, SolrRequestParserTest, TestFoldingMultitermQuery, TestTrie, SpatialFilterTest, PolyFieldTest, WordBreakSolrSpellCheckerTest, SchemaVersionSpecificBehaviorTest, SolrCoreCheckLockOnStartupTest, TestAtomicUpdateErrorCases, SortByFunctionTest, TestRemoteStreaming, TestSolrDeletionPolicy1, StandardRequestHandlerTest, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, FileBasedSpellCheckerTest, RequiredFieldsTest, TestSolrQueryParser, IndexSchemaTest, UpdateRequestProcessorFactoryTest, TestCSVResponseWriter, TestAnalyzedSuggestions, TestPartialUpdateDeduplication, TestBinaryResponseWriter, BadComponentTest, SampleTest, TestDocSet, OutputWriterTest, TestLuceneMatchVersion, SpellPossibilityIteratorTest, TestXIncludeConfig, EchoParamsTest, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestFastWriter, PluginInfoTest, ChaosMonkeyNothingIsSafeTest, DistributedIntervalFacetingTest, CollectionReloadTest, CollectionTooManyReplicasTest, DeleteNodeTest, DeleteShardTest, DeleteStatusTest, DistribDocExpirationUpdateProcessorTest, DistribJoinFromCollectionTest, DistributedQueueTest, DocValuesNotIndexedTest, LeaderFailoverAfterPartitionTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerModifyCollectionTest, OverseerStatusTest, SSLMigrationTest, SaslZkACLProviderTest, SharedFSAutoReplicaFailoverUtilsTest, SimpleCollectionCreateDeleteTest, TestCloudRecovery, TestDeleteCollectionOnDownNodes, TestLeaderElectionZkExpiry, TestLockTree, TestMiniSolrCloudCluster, TestReplicaProperties, TestTolerantUpdateProcessorRandomCloud, VMParamsZkACLAndCredentialsProvidersTest, HdfsBasicDistributedZk2Test, HdfsChaosMonkeyNothingIsSafeTest, HdfsChaosMonkeySafeLeaderTest, HdfsRecoverLeaseTest, HdfsRestartWhileUpdatingTest, HdfsSyncSliceTest, HdfsUnloadDistributedZkTest, StressHdfsTest, CoreSorterTest, ExitableDirectoryReaderTest, TestBackupRepositoryFactory, TestConfigOverlay, TestConfigSetImmutable, TestInitParams, TestReloadAndDeleteDocs, TestShardHandlerFactory, TestSolrXml, TestSolrCloudSnapshots, TestReplicationHandlerBackup, TestSolrConfigHandlerCloud, DistributedSuggestComponentTest, TestExpandComponent, TestTrackingShardHandlerFactory, HighlighterMaxOffsetTest, WrapperMergePolicyFactoryTest, JvmMetricsTest, SolrMetricsIntegrationTest, SolrGangliaReporterTest, SolrGraphiteReporterTest, SolrJmxReporterTest, SolrSlf4jReporterTest, TestIntervalFaceting, TestMacroExpander, TestMacros, TestCustomDocTransformer, TestGeoJSONResponseWriter, TestGraphMLResponseWriter, TestJavabinTupleStreamParser, TestRawResponseWriter, TestSolrQueryResponse, TestChildDocTransformer, TestBulkSchemaAPI, TestCopyFieldCollectionResource, TestDynamicFieldResource, TestSchemaSimilarityResource, TestSchemaVersionResource, TestSolrQueryParserDefaultOperatorResource, TestUniqueKeyFieldResource, TestManagedSynonymFilterFactory, BooleanFieldTest, ChangedSchemaMergeTest, EnumFieldTest, PreAnalyzedFieldManagedSchemaCloudTest, TestCloudManagedSchema, TestCloudSchemaless, TestJsonRequest, SimpleMLTQParserTest, TestPerFieldSimilarityClassic, TestPerFieldSimilarityWithDefaultOverride, TestDefaultStatsCache, TestAuthorizationFramework, ConjunctionSolrSpellCheckerTest, TestHighFrequencyDictionaryFactory, BlockCacheTest, BlockDirectoryTest, TestDocTermOrds, TestFieldCacheReopen, TestFieldCacheSanityChecker, TestFieldCacheSortRandom, TestLegacyFieldCache, TestNumericTerms32, TestNumericTerms64]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestNumericTerms64 -Dtests.seed=86D2F4C0C6ED8872 -Dtests.slow=true -Dtests.locale=ar-YE -Dtests.timezone=Etc/Greenwich -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J0 | TestNumericTerms64 (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: The test or suite printed 8944 bytes to stdout and stderr, even though the limit was set to 8192 bytes. Increase the limit with @Limit, ignore it completely with @SuppressSysoutChecks or run with -Dtests.verbose=true
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([86D2F4C0C6ED8872]:0)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4] Completed [662/681 (2!)] on J0 in 4.08s, 4 tests, 1 failure <<< FAILURES!

[...truncated 62573 lines...]


[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 634 - Still unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/634/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC

1 tests failed.
FAILED:  org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.testDelegationTokenCancelFail

Error Message:
expected:<200> but was:<404>

Stack Trace:
java.lang.AssertionError: expected:<200> but was:<404>
	at __randomizedtesting.SeedInfo.seed([4785D7EF93182F22:2F3AE2C543823DCE]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.cancelDelegationToken(TestDelegationWithHadoopAuth.java:128)
	at org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.testDelegationTokenCancelFail(TestDelegationWithHadoopAuth.java:280)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11261 lines...]
   [junit4] Suite: org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_4785D7EF93182F22-001/init-core-data-001
   [junit4]   2> 1080099 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[4785D7EF93182F22]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1080100 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[4785D7EF93182F22]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_4785D7EF93182F22-001/tempDir-001
   [junit4]   2> 1080100 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[4785D7EF93182F22]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1080104 INFO  (Thread-4345) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1080104 INFO  (Thread-4345) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1080200 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[4785D7EF93182F22]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:57006
   [junit4]   2> 1080211 INFO  (jetty-launcher-1230-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1080211 INFO  (jetty-launcher-1230-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1080214 INFO  (jetty-launcher-1230-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@283e55ea{/solr,null,AVAILABLE}
   [junit4]   2> 1080214 INFO  (jetty-launcher-1230-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1d7e6549{/solr,null,AVAILABLE}
   [junit4]   2> 1080218 INFO  (jetty-launcher-1230-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@44d96451{HTTP/1.1,[http/1.1]}{127.0.0.1:58938}
   [junit4]   2> 1080218 INFO  (jetty-launcher-1230-thread-2) [    ] o.e.j.s.Server Started @1083445ms
   [junit4]   2> 1080218 INFO  (jetty-launcher-1230-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=58938}
   [junit4]   2> 1080218 ERROR (jetty-launcher-1230-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1080218 INFO  (jetty-launcher-1230-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 1080218 INFO  (jetty-launcher-1230-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1080218 INFO  (jetty-launcher-1230-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1080218 INFO  (jetty-launcher-1230-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-25T19:48:03.591Z
   [junit4]   2> 1080219 INFO  (jetty-launcher-1230-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@898d274{HTTP/1.1,[http/1.1]}{127.0.0.1:62921}
   [junit4]   2> 1080219 INFO  (jetty-launcher-1230-thread-1) [    ] o.e.j.s.Server Started @1083446ms
   [junit4]   2> 1080219 INFO  (jetty-launcher-1230-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=62921}
   [junit4]   2> 1080219 ERROR (jetty-launcher-1230-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1080220 INFO  (jetty-launcher-1230-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 1080220 INFO  (jetty-launcher-1230-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1080220 INFO  (jetty-launcher-1230-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1080220 INFO  (jetty-launcher-1230-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-25T19:48:03.593Z
   [junit4]   2> 1080223 INFO  (jetty-launcher-1230-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1080224 INFO  (jetty-launcher-1230-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1080232 INFO  (jetty-launcher-1230-thread-2) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1080233 INFO  (jetty-launcher-1230-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57006/solr
   [junit4]   2> 1080234 INFO  (jetty-launcher-1230-thread-1) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1080235 INFO  (jetty-launcher-1230-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57006/solr
   [junit4]   2> 1080273 INFO  (jetty-launcher-1230-thread-1) [n:127.0.0.1:62921_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:62921_solr
   [junit4]   2> 1080275 INFO  (jetty-launcher-1230-thread-2) [n:127.0.0.1:58938_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58938_solr
   [junit4]   2> 1080275 INFO  (jetty-launcher-1230-thread-1) [n:127.0.0.1:62921_solr    ] o.a.s.c.Overseer Overseer (id=97345449720414213-127.0.0.1:62921_solr-n_0000000000) starting
   [junit4]   2> 1080283 INFO  (jetty-launcher-1230-thread-1) [n:127.0.0.1:62921_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62921_solr
   [junit4]   2> 1080378 INFO  (jetty-launcher-1230-thread-1) [n:127.0.0.1:62921_solr    ] o.a.s.c.CoreContainer Authentication plugin class obtained from ZK: org.apache.solr.security.HadoopAuthPlugin
   [junit4]   2> 1080384 INFO  (zkCallback-1240-thread-1-processing-n:127.0.0.1:62921_solr) [n:127.0.0.1:62921_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1080384 INFO  (zkCallback-1241-thread-1-processing-n:127.0.0.1:58938_solr) [n:127.0.0.1:58938_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1080386 INFO  (zkCallback-1241-thread-2-processing-n:127.0.0.1:58938_solr) [n:127.0.0.1:58938_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1080387 INFO  (OverseerStateUpdate-97345449720414213-127.0.0.1:62921_solr-n_0000000000) [n:127.0.0.1:62921_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1080388 INFO  (jetty-launcher-1230-thread-1) [n:127.0.0.1:62921_solr    ] o.a.s.s.HadoopAuthPlugin Params: {delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, signer.secret.provider.zookeeper.path=/token, type=simple, zk-dt-secret-manager.enable=true, signer.secret.provider=zookeeper}
   [junit4]   2> 1080473 INFO  (jetty-launcher-1230-thread-1) [n:127.0.0.1:62921_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 1080523 INFO  (jetty-launcher-1230-thread-1-EventThread) [n:127.0.0.1:62921_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 1080559 INFO  (jetty-launcher-1230-thread-2) [n:127.0.0.1:58938_solr    ] o.a.s.c.CoreContainer Authentication plugin class obtained from ZK: org.apache.solr.security.HadoopAuthPlugin
   [junit4]   2> 1080560 INFO  (jetty-launcher-1230-thread-2) [n:127.0.0.1:58938_solr    ] o.a.s.s.HadoopAuthPlugin Params: {delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, signer.secret.provider.zookeeper.path=/token, type=simple, zk-dt-secret-manager.enable=true, signer.secret.provider=zookeeper}
   [junit4]   2> 1080560 INFO  (jetty-launcher-1230-thread-2) [n:127.0.0.1:58938_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 1080564 INFO  (jetty-launcher-1230-thread-2-EventThread) [n:127.0.0.1:58938_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 1080639 INFO  (jetty-launcher-1230-thread-1) [n:127.0.0.1:62921_solr    ] o.a.s.c.CoreContainer PKIAuthenticationPlugin is managing internode requests
   [junit4]   2> 1080639 INFO  (jetty-launcher-1230-thread-1) [n:127.0.0.1:62921_solr    ] o.a.s.h.c.HttpShardHandlerFactory Reconfiguring the default client with: org.apache.solr.security.PKIAuthenticationPlugin$HttpHeaderClientConfigurer@4a259197
   [junit4]   2> 1080639 INFO  (jetty-launcher-1230-thread-1) [n:127.0.0.1:62921_solr    ] o.a.s.u.UpdateShardHandler Reconfiguring the default client with: org.apache.solr.security.PKIAuthenticationPlugin$HttpHeaderClientConfigurer@4a259197
   [junit4]   2> 1080642 INFO  (jetty-launcher-1230-thread-2) [n:127.0.0.1:58938_solr    ] o.a.s.c.CoreContainer PKIAuthenticationPlugin is managing internode requests
   [junit4]   2> 1080642 INFO  (jetty-launcher-1230-thread-2) [n:127.0.0.1:58938_solr    ] o.a.s.h.c.HttpShardHandlerFactory Reconfiguring the default client with: org.apache.solr.security.PKIAuthenticationPlugin$HttpHeaderClientConfigurer@4bc48d48
   [junit4]   2> 1080642 INFO  (jetty-launcher-1230-thread-2) [n:127.0.0.1:58938_solr    ] o.a.s.u.UpdateShardHandler Reconfiguring the default client with: org.apache.solr.security.PKIAuthenticationPlugin$HttpHeaderClientConfigurer@4bc48d48
   [junit4]   2> 1080652 INFO  (jetty-launcher-1230-thread-1) [n:127.0.0.1:62921_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_4785D7EF93182F22-001/tempDir-001/node1/.
   [junit4]   2> 1080654 INFO  (jetty-launcher-1230-thread-2) [n:127.0.0.1:58938_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_4785D7EF93182F22-001/tempDir-001/node2/.
   [junit4]   2> 1080904 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationOtherOp-seed#[4785D7EF93182F22]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationOtherOp
   [junit4]   2> 1080909 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationOtherOp-seed#[4785D7EF93182F22]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1080911 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationOtherOp-seed#[4785D7EF93182F22]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57006/solr ready
   [junit4]   2> 1080936 INFO  (qtp1642305584-10073) [n:127.0.0.1:62921_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params op=someSolrOperation&action=LIST&user.name=foo&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1080936 INFO  (qtp1642305584-10073) [n:127.0.0.1:62921_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={op=someSolrOperation&action=LIST&user.name=foo&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1080941 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationOtherOp-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=foo&p=foo&t=simple-dt&e=1485409684298&s=hT6tbOFmhvSqlW5hXwbNBXXUBzw="; Domain=127.0.0.1; Expires=jeu., 26-janv.-2017 05:48:04 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 26-janv.-2017 05:48:04 GMT
   [junit4]   2> 1080942 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationOtherOp-seed#[4785D7EF93182F22]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationOtherOp
   [junit4]   2> 1080964 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancelFail-seed#[4785D7EF93182F22]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancelFail
   [junit4]   2> 1081066 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancelFail-seed#[4785D7EF93182F22]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancelFail
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDelegationWithHadoopAuth -Dtests.method=testDelegationTokenCancelFail -Dtests.seed=4785D7EF93182F22 -Dtests.slow=true -Dtests.locale=fr-CA -Dtests.timezone=America/Bahia -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 0.13s J1 | TestDelegationWithHadoopAuth.testDelegationTokenCancelFail <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<200> but was:<404>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4785D7EF93182F22:2F3AE2C543823DCE]:0)
   [junit4]    > 	at org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.cancelDelegationToken(TestDelegationWithHadoopAuth.java:128)
   [junit4]    > 	at org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.testDelegationTokenCancelFail(TestDelegationWithHadoopAuth.java:280)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1081117 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
   [junit4]   2> 1081141 INFO  (qtp1642305584-10072) [n:127.0.0.1:62921_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7sMigFZ-zg_DAIDFK8Lm5Ug0IzL6WCUtM0uWqTy47KoB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1081141 INFO  (qtp1642305584-10072) [n:127.0.0.1:62921_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7sMigFZ-zg_DAIDFK8Lm5Ug0IzL6WCUtM0uWqTy47KoB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1081145 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1081146 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57006/solr ready
   [junit4]   2> 1081149 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081153 INFO  (qtp1771910424-10065) [n:127.0.0.1:58938_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7sMigFZ-zg_DAIDFK8Lm5Ug0IzL6WCUtM0uWqTy47KoB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1081153 INFO  (qtp1771910424-10065) [n:127.0.0.1:58938_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7sMigFZ-zg_DAIDFK8Lm5Ug0IzL6WCUtM0uWqTy47KoB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1081155 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081174 INFO  (qtp1642305584-10073) [n:127.0.0.1:62921_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7syigFZ-zg_MgMDFPZCl4HDsfnPM6WiKAxF5x9I9S2IB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1081174 INFO  (qtp1642305584-10073) [n:127.0.0.1:62921_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7syigFZ-zg_MgMDFPZCl4HDsfnPM6WiKAxF5x9I9S2IB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1081176 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081180 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1081181 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57006/solr ready
   [junit4]   2> 1081183 INFO  (qtp1642305584-10073) [n:127.0.0.1:62921_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7syigFZ-zg_MgMDFPZCl4HDsfnPM6WiKAxF5x9I9S2IB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1081184 INFO  (qtp1642305584-10073) [n:127.0.0.1:62921_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7syigFZ-zg_MgMDFPZCl4HDsfnPM6WiKAxF5x9I9S2IB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1081188 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1081190 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57006/solr ready
   [junit4]   2> 1081192 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081209 INFO  (qtp1642305584-10078) [n:127.0.0.1:62921_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7tYigFZ-zg_WAQDFIFg89eWvpR21dIWkOOV5HoPbLWFB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1081209 INFO  (qtp1642305584-10078) [n:127.0.0.1:62921_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7tYigFZ-zg_WAQDFIFg89eWvpR21dIWkOOV5HoPbLWFB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1081213 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1081214 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57006/solr ready
   [junit4]   2> 1081216 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081221 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1081223 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57006/solr ready
   [junit4]   2> 1081225 INFO  (qtp1642305584-10074) [n:127.0.0.1:62921_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7tYigFZ-zg_WAQDFIFg89eWvpR21dIWkOOV5HoPbLWFB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1081225 INFO  (qtp1642305584-10074) [n:127.0.0.1:62921_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7tYigFZ-zg_WAQDFIFg89eWvpR21dIWkOOV5HoPbLWFB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1081228 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081242 INFO  (qtp1642305584-10077) [n:127.0.0.1:62921_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7t7igFZ-zg_ewUDFFP-UJGiJuMhJAse2KrCGsxQHSzIB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1081242 INFO  (qtp1642305584-10077) [n:127.0.0.1:62921_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7t7igFZ-zg_ewUDFFP-UJGiJuMhJAse2KrCGsxQHSzIB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1081244 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081248 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1081250 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57006/solr ready
   [junit4]   2> 1081252 INFO  (qtp1642305584-10079) [n:127.0.0.1:62921_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7t7igFZ-zg_ewUDFFP-UJGiJuMhJAse2KrCGsxQHSzIB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1081252 INFO  (qtp1642305584-10079) [n:127.0.0.1:62921_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7t7igFZ-zg_ewUDFFP-UJGiJuMhJAse2KrCGsxQHSzIB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1081255 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081256 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[4785D7EF93182F22]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
   [junit4]   2> 1081283 INFO  (TEST-TestDelegationWithHadoopAuth.testZNodePaths-seed#[4785D7EF93182F22]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testZNodePaths
   [junit4]   2> 1081294 INFO  (TEST-TestDelegationWithHadoopAuth.testZNodePaths-seed#[4785D7EF93182F22]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testZNodePaths
   [junit4]   2> 1081323 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[4785D7EF93182F22]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenVerify
   [junit4]   2> 1081335 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[4785D7EF93182F22]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1081336 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[4785D7EF93182F22]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57006/solr ready
   [junit4]   2> 1081337 INFO  (qtp1642305584-10077) [n:127.0.0.1:62921_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7vbigFZ-zg_2wcDFC7t0f8LRQnGP5_TEz1IqS7PZUhjB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1081338 INFO  (qtp1642305584-10077) [n:127.0.0.1:62921_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7vbigFZ-zg_2wcDFC7t0f8LRQnGP5_TEz1IqS7PZUhjB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1081343 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[4785D7EF93182F22]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1081345 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[4785D7EF93182F22]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57006/solr ready
   [junit4]   2> 1081347 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081351 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[4785D7EF93182F22]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1081353 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[4785D7EF93182F22]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57006/solr ready
   [junit4]   2> 1081355 INFO  (qtp1642305584-10074) [n:127.0.0.1:62921_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7vbigFZ-zg_2wcDFC7t0f8LRQnGP5_TEz1IqS7PZUhjB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1081355 INFO  (qtp1642305584-10074) [n:127.0.0.1:62921_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7vbigFZ-zg_2wcDFC7t0f8LRQnGP5_TEz1IqS7PZUhjB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1081360 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[4785D7EF93182F22]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1081361 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[4785D7EF93182F22]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57006/solr ready
   [junit4]   2> 1081363 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081365 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[4785D7EF93182F22]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenVerify
   [junit4]   2> 1081396 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[4785D7EF93182F22]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancel
   [junit4]   2> 1081411 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[4785D7EF93182F22]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1081413 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[4785D7EF93182F22]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57006/solr ready
   [junit4]   2> 1081415 ERROR (qtp1642305584-10078) [n:127.0.0.1:62921_solr    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_8]
   [junit4]   2> 1081422 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081427 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[4785D7EF93182F22]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1081428 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[4785D7EF93182F22]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57006/solr ready
   [junit4]   2> 1081430 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081446 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[4785D7EF93182F22]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1081448 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[4785D7EF93182F22]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57006/solr ready
   [junit4]   2> 1081450 ERROR (qtp1642305584-10074) [n:127.0.0.1:62921_solr    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_9]
   [junit4]   2> 1081457 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[4785D7EF93182F22]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1081459 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[4785D7EF93182F22]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57006/solr ready
   [junit4]   2> 1081461 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081464 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081465 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[4785D7EF93182F22]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancel
   [junit4]   2> 1081490 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenSolrClient-seed#[4785D7EF93182F22]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenSolrClient
   [junit4]   2> 1081502 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenSolrClient-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081504 INFO  (qtp1642305584-10080) [n:127.0.0.1:62921_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7yFigFZ-zhAhQoDFJe1DWC3-mwH-Q9kCO80FRQ_WQrGB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1081504 INFO  (qtp1642305584-10080) [n:127.0.0.1:62921_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnXK7yFigFZ-zhAhQoDFJe1DWC3-mwH-Q9kCO80FRQ_WQrGB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1081505 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenSolrClient-seed#[4785D7EF93182F22]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenSolrClient
   [junit4]   2> 1081535 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[4785D7EF93182F22]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
   [junit4]   2> 1081546 WARN  (qtp1642305584-10073) [n:127.0.0.1:62921_solr    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: bar tries to renew a token with renewer foo
   [junit4]   2> 1081547 WARN  (qtp1642305584-10073) [n:127.0.0.1:62921_solr    ] o.a.s.s.HadoopAuthPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 1081547 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081549 WARN  (qtp1771910424-10063) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: bar tries to renew a token with renewer foo
   [junit4]   2> 1081549 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081549 WARN  (qtp1771910424-10063) [    ] o.a.s.s.HadoopAuthPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 1081560 WARN  (qtp1642305584-10078) [n:127.0.0.1:62921_solr    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 1081561 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081561 WARN  (qtp1642305584-10078) [n:127.0.0.1:62921_solr    ] o.a.s.s.HadoopAuthPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 1081564 WARN  (qtp1771910424-10066) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 1081564 WARN  (qtp1771910424-10066) [    ] o.a.s.s.HadoopAuthPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 1081564 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[4785D7EF93182F22]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires=jeu., 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: jeu., 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 1081565 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[4785D7EF93182F22]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenewFail
   [junit4]   2> 1081567 INFO  (jetty-launcher-1230-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@44d96451{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1081567 INFO  (jetty-launcher-1230-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=2140763823
   [junit4]   2> 1081570 INFO  (jetty-launcher-1230-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@898d274{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1081571 INFO  (jetty-launcher-1230-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1186326535
   [junit4]   2> 1081582 INFO  (jetty-launcher-1230-thread-2) [    ] o.a.s.c.Overseer Overseer (id=97345449720414213-127.0.0.1:62921_solr-n_0000000000) closing
   [junit4]   2> 1081582 INFO  (OverseerStateUpdate-97345449720414213-127.0.0.1:62921_solr-n_0000000000) [n:127.0.0.1:62921_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:62921_solr
   [junit4]   2> 1081582 INFO  (zkCallback-1240-thread-3-processing-n:127.0.0.1:62921_solr) [n:127.0.0.1:62921_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 1081583 INFO  (jetty-launcher-1230-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1081584 ERROR (Thread[Thread-4362,5,TGRP-TestDelegationWithHadoopAuth]) [n:127.0.0.1:62921_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 1081595 INFO  (jetty-launcher-1230-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1d7e6549{/solr,null,UNAVAILABLE}
   [junit4]   2> 1083081 WARN  (zkCallback-1241-thread-2-processing-n:127.0.0.1:58938_solr) [n:127.0.0.1:58938_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1083082 INFO  (jetty-launcher-1230-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1083082 ERROR (Thread[Thread-4364,5,TGRP-TestDelegationWithHadoopAuth]) [n:127.0.0.1:58938_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 1083087 INFO  (jetty-launcher-1230-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@283e55ea{/solr,null,UNAVAILABLE}
   [junit4]   2> 1083089 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[4785D7EF93182F22]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:57006 57006
   [junit4]   2> 1083126 INFO  (Thread-4345) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:57006 57006
   [junit4]   2> 1083127 WARN  (Thread-4345) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	17	/solr/aliases.json
   [junit4]   2> 	17	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	17	/solr/clusterstate.json
   [junit4]   2> 	4	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_7
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_6
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_9
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_8
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_5
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_12
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_11
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_10
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	17	/solr/live_nodes
   [junit4]   2> 	17	/solr/collections
   [junit4]   2> 	3	/solr/overseer/queue
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 
   [junit4]   2> 1083128 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[4785D7EF93182F22]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_4785D7EF93182F22-001
   [junit4]   2> Jan 25, 2017 7:48:06 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=1670, maxMBSortInHeap=5.125795871975305, sim=RandomSimilarity(queryNorm=false,coord=crazy): {}, locale=fr-CA, timezone=America/Bahia
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=62429688,total=402653184
   [junit4]   2> NOTE: All tests run in this JVM: [SimpleMLTQParserTest, BooleanFieldTest, DistributedDebugComponentTest, TestQuerySenderNoQuery, TestJmxMonitoredMap, MetricsHandlerTest, TestSolrQueryParser, TestTrackingShardHandlerFactory, SolrCoreCheckLockOnStartupTest, ResponseHeaderTest, TestReqParamsAPI, TestStressUserVersions, URLClassifyProcessorTest, CoreAdminCreateDiscoverTest, OverseerTaskQueueTest, TestSolr4Spatial2, SpatialHeatmapFacetsTest, TestSQLHandler, TestHdfsUpdateLog, DistanceFunctionTest, HdfsDirectoryTest, SSLMigrationTest, TestMacroExpander, DistribJoinFromCollectionTest, BasicAuthIntegrationTest, BlockDirectoryTest, RestartWhileUpdatingTest, DeleteInactiveReplicaTest, StatsComponentTest, TestExpandComponent, RegexBoostProcessorTest, TestXmlQParser, TestCollectionAPI, TestJoin, ShardSplitTest, WordBreakSolrSpellCheckerTest, SpellPossibilityIteratorTest, ClassificationUpdateProcessorTest, TestGraphTermsQParserPlugin, TestDFISimilarityFactory, CollectionsAPIDistributedZkTest, TestCoreDiscovery, CacheHeaderTest, SolrCLIZkUtilsTest, TestClusterStateMutator, DefaultValueUpdateProcessorTest, CSVRequestHandlerTest, CheckHdfsIndexTest, TestSuggestSpellingConverter, SortByFunctionTest, RecoveryAfterSoftCommitTest, TestJsonRequest, ForceLeaderTest, OverseerRolesTest, TestMissingGroups, ClassificationUpdateProcessorFactoryTest, ShardRoutingCustomTest, AnalyticsMergeStrategyTest, TestIndexingPerformance, DebugComponentTest, TestSearchPerf, SuggesterTest, TestFiltering, LeaderFailureAfterFreshStartTest, TestConfigSetImmutable, BlockJoinFacetDistribTest, TestInitQParser, TestDelegationWithHadoopAuth]
   [junit4] Completed [172/681 (1!)] on J1 in 3.55s, 8 tests, 1 failure <<< FAILURES!

[...truncated 64154 lines...]


[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 633 - Failure!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/633/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail

Error Message:
expected:<200> but was:<404>

Stack Trace:
java.lang.AssertionError: expected:<200> but was:<404>
	at __randomizedtesting.SeedInfo.seed([F39113DC53B2730A:9B2E26F6832861E6]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:141)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:295)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10773 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_F39113DC53B2730A-001/init-core-data-001
   [junit4]   2> 30372 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[F39113DC53B2730A]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 30374 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[F39113DC53B2730A]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_F39113DC53B2730A-001/tempDir-001
   [junit4]   2> 30374 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[F39113DC53B2730A]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 30375 INFO  (Thread-134) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 30375 INFO  (Thread-134) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 30475 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[F39113DC53B2730A]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:56872
   [junit4]   2> 30491 INFO  (jetty-launcher-55-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 30493 INFO  (jetty-launcher-55-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5b940d6f{/solr,null,AVAILABLE}
   [junit4]   2> 30494 INFO  (jetty-launcher-55-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@28f143a0{HTTP/1.1,[http/1.1]}{127.0.0.1:64845}
   [junit4]   2> 30494 INFO  (jetty-launcher-55-thread-2) [    ] o.e.j.s.Server Started @32988ms
   [junit4]   2> 30494 INFO  (jetty-launcher-55-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=64845}
   [junit4]   2> 30494 ERROR (jetty-launcher-55-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 30494 INFO  (jetty-launcher-55-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.5.0
   [junit4]   2> 30495 INFO  (jetty-launcher-55-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 30495 INFO  (jetty-launcher-55-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 30495 INFO  (jetty-launcher-55-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-25T07:42:52.594Z
   [junit4]   2> 30497 INFO  (jetty-launcher-55-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 30503 INFO  (jetty-launcher-55-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@676d2117{/solr,null,AVAILABLE}
   [junit4]   2> 30503 INFO  (jetty-launcher-55-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3344bd94{HTTP/1.1,[http/1.1]}{127.0.0.1:40101}
   [junit4]   2> 30503 INFO  (jetty-launcher-55-thread-1) [    ] o.e.j.s.Server Started @32998ms
   [junit4]   2> 30503 INFO  (jetty-launcher-55-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40101}
   [junit4]   2> 30504 ERROR (jetty-launcher-55-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 30504 INFO  (jetty-launcher-55-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.5.0
   [junit4]   2> 30504 INFO  (jetty-launcher-55-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 30504 INFO  (jetty-launcher-55-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 30504 INFO  (jetty-launcher-55-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-25T07:42:52.603Z
   [junit4]   2> 30518 INFO  (jetty-launcher-55-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 30518 INFO  (jetty-launcher-55-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 30528 INFO  (jetty-launcher-55-thread-1) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 30529 INFO  (jetty-launcher-55-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56872/solr
   [junit4]   2> 30530 INFO  (jetty-launcher-55-thread-2) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 30532 INFO  (jetty-launcher-55-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56872/solr
   [junit4]   2> 30609 INFO  (jetty-launcher-55-thread-2) [n:127.0.0.1:64845_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:64845_solr
   [junit4]   2> 30612 INFO  (jetty-launcher-55-thread-2) [n:127.0.0.1:64845_solr    ] o.a.s.c.Overseer Overseer (id=97342598183321606-127.0.0.1:64845_solr-n_0000000000) starting
   [junit4]   2> 30616 INFO  (jetty-launcher-55-thread-1) [n:127.0.0.1:40101_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40101_solr
   [junit4]   2> 30620 INFO  (zkCallback-66-thread-1-processing-n:127.0.0.1:64845_solr) [n:127.0.0.1:64845_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 30627 INFO  (jetty-launcher-55-thread-2) [n:127.0.0.1:64845_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64845_solr
   [junit4]   2> 30788 INFO  (jetty-launcher-55-thread-1) [n:127.0.0.1:40101_solr    ] o.a.s.c.CoreContainer Authentication plugin class obtained from system property 'authenticationPlugin': org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 30834 INFO  (zkCallback-65-thread-2-processing-n:127.0.0.1:40101_solr) [n:127.0.0.1:40101_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 30838 INFO  (OverseerStateUpdate-97342598183321606-127.0.0.1:64845_solr-n_0000000000) [n:127.0.0.1:64845_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 30842 INFO  (jetty-launcher-55-thread-2) [n:127.0.0.1:64845_solr    ] o.a.s.c.CoreContainer Authentication plugin class obtained from system property 'authenticationPlugin': org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 30842 INFO  (jetty-launcher-55-thread-1) [n:127.0.0.1:40101_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
   [junit4]   2> 30842 INFO  (jetty-launcher-55-thread-2) [n:127.0.0.1:64845_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
   [junit4]   2> 30953 INFO  (jetty-launcher-55-thread-1) [n:127.0.0.1:40101_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 30961 INFO  (jetty-launcher-55-thread-2) [n:127.0.0.1:64845_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 30994 INFO  (jetty-launcher-55-thread-1-EventThread) [n:127.0.0.1:40101_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 30997 INFO  (jetty-launcher-55-thread-2-EventThread) [n:127.0.0.1:64845_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 32267 INFO  (jetty-launcher-55-thread-1) [n:127.0.0.1:40101_solr    ] o.a.s.h.c.HttpShardHandlerFactory Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@61ea4870
   [junit4]   2> 32267 INFO  (jetty-launcher-55-thread-1) [n:127.0.0.1:40101_solr    ] o.a.s.u.UpdateShardHandler Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@61ea4870
   [junit4]   2> 32271 INFO  (jetty-launcher-55-thread-2) [n:127.0.0.1:64845_solr    ] o.a.s.h.c.HttpShardHandlerFactory Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@49f834d
   [junit4]   2> 32271 INFO  (jetty-launcher-55-thread-2) [n:127.0.0.1:64845_solr    ] o.a.s.u.UpdateShardHandler Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@49f834d
   [junit4]   2> 32274 INFO  (jetty-launcher-55-thread-2) [n:127.0.0.1:64845_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_F39113DC53B2730A-001/tempDir-001/node1/.
   [junit4]   2> 32275 INFO  (jetty-launcher-55-thread-1) [n:127.0.0.1:40101_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_F39113DC53B2730A-001/tempDir-001/node2/.
   [junit4]   2> 32345 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[F39113DC53B2730A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
   [junit4]   2> 33096 WARN  (qtp33834744-397) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
   [junit4]   2> 33112 WARN  (qtp33834744-397) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 33136 WARN  (qtp1418050088-407) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
   [junit4]   2> 33138 WARN  (qtp1418050088-407) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 33181 WARN  (qtp33834744-402) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 33181 WARN  (qtp33834744-402) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 33185 WARN  (qtp1418050088-409) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 33186 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[F39113DC53B2730A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenewFail
   [junit4]   2> 33187 WARN  (qtp1418050088-409) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 33210 INFO  (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[F39113DC53B2730A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testZNodePaths
   [junit4]   2> 33232 INFO  (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[F39113DC53B2730A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testZNodePaths
   [junit4]   2> 33251 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[F39113DC53B2730A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancelFail
   [junit4]   2> 33337 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[F39113DC53B2730A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancelFail
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithDelegationTokens -Dtests.method=testDelegationTokenCancelFail -Dtests.seed=F39113DC53B2730A -Dtests.slow=true -Dtests.locale=en-NZ -Dtests.timezone=America/Belize -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.11s J0 | TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<200> but was:<404>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F39113DC53B2730A:9B2E26F6832861E6]:0)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:141)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:295)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 33364 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[F39113DC53B2730A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenVerify
   [junit4]   2> 33403 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[F39113DC53B2730A]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 33405 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[F39113DC53B2730A]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56872/solr ready
   [junit4]   2> 33422 INFO  (qtp33834744-392) [n:127.0.0.1:64845_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9XqigFZ-KBZ6gUDFH527ifXXTggwaWzjWqUGY0rLkKXB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 33422 INFO  (qtp33834744-392) [n:127.0.0.1:64845_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9XqigFZ-KBZ6gUDFH527ifXXTggwaWzjWqUGY0rLkKXB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 33452 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[F39113DC53B2730A]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 33455 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[F39113DC53B2730A]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56872/solr ready
   [junit4]   2> 33457 INFO  (qtp33834744-399) [n:127.0.0.1:64845_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9XqigFZ-KBZ6gUDFH527ifXXTggwaWzjWqUGY0rLkKXB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 33458 INFO  (qtp33834744-399) [n:127.0.0.1:64845_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9XqigFZ-KBZ6gUDFH527ifXXTggwaWzjWqUGY0rLkKXB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 33466 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[F39113DC53B2730A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenVerify
   [junit4]   2> 33497 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[F39113DC53B2730A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenSolrClient
   [junit4]   2> 33562 INFO  (qtp33834744-398) [n:127.0.0.1:64845_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9Z2igFZ-KBadgYDFM6x1YiRs8aJsOVdLA9i-YS7pVumB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 33562 INFO  (qtp33834744-398) [n:127.0.0.1:64845_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9Z2igFZ-KBadgYDFM6x1YiRs8aJsOVdLA9i-YS7pVumB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 33563 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[F39113DC53B2730A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenSolrClient
   [junit4]   2> 33605 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[F39113DC53B2730A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
   [junit4]   2> 33677 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[F39113DC53B2730A]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 33678 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[F39113DC53B2730A]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56872/solr ready
   [junit4]   2> 33682 INFO  (qtp33834744-393) [n:127.0.0.1:64845_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9bfigFZ-KBa3wcDFFqfk1VOQ2QkhnwBTBMg88iza4qMB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 33682 INFO  (qtp33834744-393) [n:127.0.0.1:64845_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9bfigFZ-KBa3wcDFFqfk1VOQ2QkhnwBTBMg88iza4qMB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 33691 INFO  (qtp1418050088-414) [n:127.0.0.1:40101_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9bfigFZ-KBa3wcDFFqfk1VOQ2QkhnwBTBMg88iza4qMB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 33691 INFO  (qtp1418050088-414) [n:127.0.0.1:40101_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9bfigFZ-KBa3wcDFFqfk1VOQ2QkhnwBTBMg88iza4qMB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 33763 INFO  (qtp33834744-406) [n:127.0.0.1:64845_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9dPigFZ-KBbTwgDFDoQwpcc63JVhwiXKQrXYQ2H0lTWB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 33763 INFO  (qtp33834744-406) [n:127.0.0.1:64845_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9dPigFZ-KBbTwgDFDoQwpcc63JVhwiXKQrXYQ2H0lTWB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 33772 INFO  (qtp1418050088-409) [n:127.0.0.1:40101_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9dPigFZ-KBbTwgDFDoQwpcc63JVhwiXKQrXYQ2H0lTWB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 33772 INFO  (qtp1418050088-409) [n:127.0.0.1:40101_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9dPigFZ-KBbTwgDFDoQwpcc63JVhwiXKQrXYQ2H0lTWB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 33822 INFO  (qtp33834744-399) [n:127.0.0.1:64845_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9eJigFZ-KBbiQkDFBVCoZcxcvYO14dUYE0WfVZhz71eB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 33822 INFO  (qtp33834744-399) [n:127.0.0.1:64845_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9eJigFZ-KBbiQkDFBVCoZcxcvYO14dUYE0WfVZhz71eB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 33850 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[F39113DC53B2730A]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 33852 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[F39113DC53B2730A]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56872/solr ready
   [junit4]   2> 33876 INFO  (qtp1418050088-411) [n:127.0.0.1:40101_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9eJigFZ-KBbiQkDFBVCoZcxcvYO14dUYE0WfVZhz71eB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 33877 INFO  (qtp1418050088-411) [n:127.0.0.1:40101_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9eJigFZ-KBbiQkDFBVCoZcxcvYO14dUYE0WfVZhz71eB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 33891 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[F39113DC53B2730A]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 33894 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[F39113DC53B2730A]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56872/solr ready
   [junit4]   2> 33977 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[F39113DC53B2730A]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 33979 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[F39113DC53B2730A]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56872/solr ready
   [junit4]   2> 34002 INFO  (qtp33834744-406) [n:127.0.0.1:64845_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9gIigFZ-KBcCAoDFIHxCcCK0-U7YjRS7L2LZ07OvFh3B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 34002 INFO  (qtp33834744-406) [n:127.0.0.1:64845_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9gIigFZ-KBcCAoDFIHxCcCK0-U7YjRS7L2LZ07OvFh3B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 34011 INFO  (qtp1418050088-414) [n:127.0.0.1:40101_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9gIigFZ-KBcCAoDFIHxCcCK0-U7YjRS7L2LZ07OvFh3B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 34011 INFO  (qtp1418050088-414) [n:127.0.0.1:40101_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnUk9gIigFZ-KBcCAoDFIHxCcCK0-U7YjRS7L2LZ07OvFh3B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 34015 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[F39113DC53B2730A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
   [junit4]   2> 34034 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[F39113DC53B2730A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationOtherOp
   [junit4]   2> 34049 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[F39113DC53B2730A]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 34050 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[F39113DC53B2730A]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56872/solr ready
   [junit4]   2> 34070 INFO  (qtp33834744-406) [n:127.0.0.1:64845_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params op=someSolrOperation&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 34071 INFO  (qtp33834744-406) [n:127.0.0.1:64845_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={op=someSolrOperation&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 34095 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[F39113DC53B2730A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationOtherOp
   [junit4]   2> 34128 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[F39113DC53B2730A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancel
   [junit4]   2> 34151 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[F39113DC53B2730A]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 34152 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[F39113DC53B2730A]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56872/solr ready
   [junit4]   2> 34159 ERROR (qtp33834744-392) [n:127.0.0.1:64845_solr    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_11]
   [junit4]   2> 34258 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[F39113DC53B2730A]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 34260 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[F39113DC53B2730A]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56872/solr ready
   [junit4]   2> 34305 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[F39113DC53B2730A]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 34307 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[F39113DC53B2730A]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56872/solr ready
   [junit4]   2> 34345 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[F39113DC53B2730A]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 34346 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[F39113DC53B2730A]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56872/solr ready
   [junit4]   2> 34352 ERROR (qtp33834744-403) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_12]
   [junit4]   2> 34386 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[F39113DC53B2730A]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 34387 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[F39113DC53B2730A]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56872/solr ready
   [junit4]   2> 34392 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[F39113DC53B2730A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancel
   [junit4]   2> 34395 INFO  (jetty-launcher-55-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@28f143a0{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 34395 INFO  (jetty-launcher-55-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1155429809
   [junit4]   2> 34414 INFO  (jetty-launcher-55-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@3344bd94{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 34415 INFO  (jetty-launcher-55-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1377277192
   [junit4]   2> 34415 INFO  (jetty-launcher-55-thread-1) [    ] o.a.s.c.Overseer Overseer (id=97342598183321606-127.0.0.1:64845_solr-n_0000000000) closing
   [junit4]   2> 34418 INFO  (OverseerStateUpdate-97342598183321606-127.0.0.1:64845_solr-n_0000000000) [n:127.0.0.1:64845_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:64845_solr
   [junit4]   2> 34433 WARN  (zkCallback-66-thread-3-processing-n:127.0.0.1:64845_solr) [n:127.0.0.1:64845_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 34433 INFO  (jetty-launcher-55-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 34434 ERROR (Thread[Thread-154,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:64845_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 34436 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159d493ca3b0006, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 34450 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159d493ca3b0005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 34452 INFO  (jetty-launcher-55-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5b940d6f{/solr,null,UNAVAILABLE}
   [junit4]   2> 35938 WARN  (zkCallback-65-thread-2-processing-n:127.0.0.1:40101_solr) [n:127.0.0.1:40101_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 35939 INFO  (jetty-launcher-55-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 35939 ERROR (Thread[Thread-156,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:40101_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 35942 INFO  (jetty-launcher-55-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@676d2117{/solr,null,UNAVAILABLE}
   [junit4]   2> 35945 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[F39113DC53B2730A]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:56872 56872
   [junit4]   2> 35960 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxnFactory Ignoring unexpected runtime exception
   [junit4]   2> java.nio.channels.CancelledKeyException
   [junit4]   2> 	at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73)
   [junit4]   2> 	at sun.nio.ch.SelectionKeyImpl.readyOps(SelectionKeyImpl.java:87)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:187)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 35963 INFO  (Thread-134) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:56872 56872
   [junit4]   2> 35966 WARN  (Thread-134) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	14	/solr/aliases.json
   [junit4]   2> 	14	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	14	/solr/clusterstate.json
   [junit4]   2> 	4	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_7
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_6
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_9
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_8
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_5
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_12
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_10
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	14	/solr/live_nodes
   [junit4]   2> 	14	/solr/collections
   [junit4]   2> 	3	/solr/overseer/queue
   [junit4]   2> 	3	/solr/overseer/collection-queue-work
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 
   [junit4]   2> 35967 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[F39113DC53B2730A]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_F39113DC53B2730A-001
   [junit4]   2> Jan 25, 2017 7:42:58 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=2, maxDocsPerChunk=742, blockSize=6), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=2, blockSize=6)), sim=RandomSimilarity(queryNorm=true,coord=no): {}, locale=en-NZ, timezone=America/Belize
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=54375712,total=134258688
   [junit4]   2> NOTE: All tests run in this JVM: [NumericFieldsTest, TestDefaultStatsCache, PathHierarchyTokenizerFactoryTest, DeleteLastCustomShardedReplicaTest, TestSolrQueryResponse, QueryResultKeyTest, TestSolrCoreParser, ConjunctionSolrSpellCheckerTest, RequiredFieldsTest, TestClusterProperties, BadCopyFieldTest, TestSolrCloudWithDelegationTokens]
   [junit4] Completed [16/681 (1!)] on J0 in 6.63s, 8 tests, 1 failure <<< FAILURES!

[...truncated 54743 lines...]
   [junit4] ERROR: JVM J0 ended with an exception, command line: /usr/jdk/instances/jdk1.8.0/jre/bin/java -XX:+UseCompressedOops -XX:+UseSerialGC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/heapdumps -ea -esa -Dtests.prefix=tests -Dtests.seed=F39113DC53B2730A -Xmx512M -Dtests.iters= -Dtests.verbose=false -Dtests.infostream=false -Dtests.codec=random -Dtests.postingsformat=random -Dtests.docvaluesformat=random -Dtests.locale=random -Dtests.timezone=random -Dtests.directory=random -Dtests.linedocsfile=europarl.lines.txt.gz -Dtests.luceneMatchVersion=6.5.0 -Dtests.cleanthreads=perClass -Djava.util.logging.config.file=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/tools/junit4/logging.properties -Dtests.nightly=false -Dtests.weekly=false -Dtests.monster=false -Dtests.slow=true -Dtests.asserts=true -Dtests.multiplier=1 -DtempDir=./temp -Djava.io.tmpdir=./temp -Djunit4.tempDir=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/temp -Dcommon.dir=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene -Dclover.db.dir=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/clover/db -Djava.security.policy=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/tools/junit4/solr-tests.policy -Dtests.LUCENE_VERSION=6.5.0 -Djetty.testMode=1 -Djetty.insecurerandom=1 -Dsolr.directoryFactory=org.apache.solr.core.MockDirectoryFactory -Djava.awt.headless=true -Djdk.map.althashing.threshold=0 -Djunit4.childvm.cwd=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0 -Djunit4.childvm.id=0 -Djunit4.childvm.count=2 -Dtests.leaveTemporary=false -Dtests.filterstacks=true -Djava.security.manager=org.apache.lucene.util.TestSecurityManager -classpath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/classes/test:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-test-framework/classes/java:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/test-framework/lib/junit4-ant-2.5.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/test-framework/classes/java:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/codecs/classes/java:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/classes/java:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/classes/java:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/analysis/common/lucene-analyzers-common-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/analysis/kuromoji/lucene-analyzers-kuromoji-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/analysis/phonetic/lucene-analyzers-phonetic-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/codecs/lucene-codecs-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/backward-codecs/lucene-backward-codecs-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/highlighter/lucene-highlighter-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/memory/lucene-memory-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/misc/lucene-misc-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/spatial-extras/lucene-spatial-extras-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/expressions/lucene-expressions-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/suggest/lucene-suggest-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/grouping/lucene-grouping-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/queries/lucene-queries-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/queryparser/lucene-queryparser-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/join/lucene-join-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/sandbox/lucene-sandbox-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/classification/lucene-classification-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/antlr4-runtime-4.5.1-1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/asm-5.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/asm-commons-5.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/caffeine-1.0.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/commons-cli-1.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/commons-codec-1.10.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/commons-collections-3.2.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/commons-configuration-1.6.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/commons-exec-1.3.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/commons-fileupload-1.3.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/commons-io-2.5.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/commons-lang-2.6.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/curator-client-2.8.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/curator-framework-2.8.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/curator-recipes-2.8.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/dom4j-1.6.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/guava-14.0.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/hadoop-annotations-2.7.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/hadoop-auth-2.7.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/hadoop-common-2.7.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/hadoop-hdfs-2.7.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/hppc-0.7.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/htrace-core-3.2.0-incubating.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/jackson-core-2.5.4.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/jackson-core-asl-1.9.13.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/jackson-dataformat-smile-2.5.4.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/jackson-mapper-asl-1.9.13.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/jcl-over-slf4j-1.7.7.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/joda-time-2.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/log4j-1.2.17.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/org.restlet-2.3.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/org.restlet.ext.servlet-2.3.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/presto-parser-0.122.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/protobuf-java-2.5.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/slf4j-log4j12-1.7.7.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/slice-0.10.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/spatial4j-0.6.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/t-digest-3.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/commons-io-2.5.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/httpclient-4.4.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/httpcore-4.4.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/httpmime-4.4.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/jcl-over-slf4j-1.7.7.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/noggit-0.6.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/slf4j-api-1.7.7.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/stax2-api-3.1.4.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/woodstox-core-asl-4.4.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/zookeeper-3.4.6.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/gmetric4j-1.0.7.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/javax.servlet-api-3.1.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-continuation-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-deploy-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-http-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-io-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-jmx-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-rewrite-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-security-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-server-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-servlet-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-servlets-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-util-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-webapp-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-xml-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/metrics-core-3.1.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/metrics-ganglia-3.1.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/metrics-graphite-3.1.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/metrics-jetty9-3.1.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/metrics-jvm-3.1.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/example/example-DIH/solr/db/lib/derby-10.9.1.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/example/example-DIH/solr/db/lib/hsqldb-1.8.0.10.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/core/classes/java:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/test-framework/lib/junit-4.10.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/test-framework/lib/randomizedtesting-runner-2.5.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/antlr-2.7.7.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/antlr4-runtime-4.5.1-1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-core-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-core-api-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-core-avl-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-core-shared-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-i18n-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptor-kerberos-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-admin-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-authn-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-authz-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/api-all-1.0.0-M20.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/bcprov-jdk15-1.45.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/byte-buddy-1.6.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/cglib-nodep-2.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/commons-collections-3.2.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/commons-math3-3.4.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/easymock-3.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/ehcache-core-2.4.4.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/hadoop-common-2.7.2-tests.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/hadoop-hdfs-2.7.2-tests.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/hadoop-minikdc-2.7.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/jackson-annotations-2.5.4.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/jackson-databind-2.5.4.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/jersey-core-1.9.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/jersey-server-1.9.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/jetty-6.1.26.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/jetty-util-6.1.26.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/mina-core-2.0.0-M5.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/mockito-core-2.6.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/netty-all-4.0.36.Final.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/objenesis-2.5.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/presto-parser-0.122.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/slice-0.10.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/analysis/icu/lucene-analyzers-icu-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/contrib/solr-analysis-extras/classes/java:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/queryparser/classes/test:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/contrib/analysis-extras/lib/icu4j-56.1.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-launcher.jar:/export/home/jenkins/.ant/lib/ivy-2.3.0.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-commons-logging.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jai.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-oro.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-swing.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-netrexx.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-bcel.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-regexp.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-bsf.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jsch.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-junit.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-antlr.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-commons-net.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jmf.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jdepend.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-junit4.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-javamail.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-xalan2.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-log4j.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-resolver.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-testutil.jar:/usr/jdk/instances/jdk1.8.0/lib/tools.jar:/export/home/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.0.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/temp/junit4-J0-20170125_074219_5146564980560994785231.events @/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/temp/junit4-J0-20170125_074219_5144777017464731072575.suites -stdin
   [junit4] ERROR: JVM J0 ended with an exception: java.io.IOException: No space left on device
   [junit4] 	at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
   [junit4] 	at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60)
   [junit4] 	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
   [junit4] 	at sun.nio.ch.IOUtil.write(IOUtil.java:65)
   [junit4] 	at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211)
   [junit4] 	at java.nio.channels.Channels.writeFullyImpl(Channels.java:78)
   [junit4] 	at java.nio.channels.Channels.writeFully(Channels.java:101)
   [junit4] 	at java.nio.channels.Channels.access$000(Channels.java:61)
   [junit4] 	at java.nio.channels.Channels$1.write(Channels.java:174)
   [junit4] 	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
   [junit4] 	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
   [junit4] 	at java.io.FilterOutputStream.close(FilterOutputStream.java:158)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.io.Closer.close(Closer.java:214)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1526)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4] 	at java.lang.Thread.run(Thread.java:745)
   [junit4] ERROR: JVM J1 ended with an exception, command line: /usr/jdk/instances/jdk1.8.0/jre/bin/java -XX:+UseCompressedOops -XX:+UseSerialGC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/heapdumps -ea -esa -Dtests.prefix=tests -Dtests.seed=F39113DC53B2730A -Xmx512M -Dtests.iters= -Dtests.verbose=false -Dtests.infostream=false -Dtests.codec=random -Dtests.postingsformat=random -Dtests.docvaluesformat=random -Dtests.locale=random -Dtests.timezone=random -Dtests.directory=random -Dtests.linedocsfile=europarl.lines.txt.gz -Dtests.luceneMatchVersion=6.5.0 -Dtests.cleanthreads=perClass -Djava.util.logging.config.file=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/tools/junit4/logging.properties -Dtests.nightly=false -Dtests.weekly=false -Dtests.monster=false -Dtests.slow=true -Dtests.asserts=true -Dtests.multiplier=1 -DtempDir=./temp -Djava.io.tmpdir=./temp -Djunit4.tempDir=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/temp -Dcommon.dir=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene -Dclover.db.dir=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/clover/db -Djava.security.policy=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/tools/junit4/solr-tests.policy -Dtests.LUCENE_VERSION=6.5.0 -Djetty.testMode=1 -Djetty.insecurerandom=1 -Dsolr.directoryFactory=org.apache.solr.core.MockDirectoryFactory -Djava.awt.headless=true -Djdk.map.althashing.threshold=0 -Djunit4.childvm.cwd=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1 -Djunit4.childvm.id=1 -Djunit4.childvm.count=2 -Dtests.leaveTemporary=false -Dtests.filterstacks=true -Djava.security.manager=org.apache.lucene.util.TestSecurityManager -classpath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/classes/test:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-test-framework/classes/java:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/test-framework/lib/junit4-ant-2.5.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/test-framework/classes/java:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/codecs/classes/java:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-solrj/classes/java:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/classes/java:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/analysis/common/lucene-analyzers-common-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/analysis/kuromoji/lucene-analyzers-kuromoji-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/analysis/phonetic/lucene-analyzers-phonetic-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/codecs/lucene-codecs-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/backward-codecs/lucene-backward-codecs-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/highlighter/lucene-highlighter-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/memory/lucene-memory-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/misc/lucene-misc-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/spatial-extras/lucene-spatial-extras-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/expressions/lucene-expressions-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/suggest/lucene-suggest-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/grouping/lucene-grouping-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/queries/lucene-queries-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/queryparser/lucene-queryparser-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/join/lucene-join-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/sandbox/lucene-sandbox-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/classification/lucene-classification-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/antlr4-runtime-4.5.1-1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/asm-5.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/asm-commons-5.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/caffeine-1.0.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/commons-cli-1.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/commons-codec-1.10.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/commons-collections-3.2.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/commons-configuration-1.6.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/commons-exec-1.3.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/commons-fileupload-1.3.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/commons-io-2.5.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/commons-lang-2.6.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/curator-client-2.8.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/curator-framework-2.8.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/curator-recipes-2.8.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/dom4j-1.6.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/guava-14.0.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/hadoop-annotations-2.7.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/hadoop-auth-2.7.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/hadoop-common-2.7.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/hadoop-hdfs-2.7.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/hppc-0.7.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/htrace-core-3.2.0-incubating.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/jackson-core-2.5.4.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/jackson-core-asl-1.9.13.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/jackson-dataformat-smile-2.5.4.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/jackson-mapper-asl-1.9.13.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/jcl-over-slf4j-1.7.7.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/joda-time-2.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/log4j-1.2.17.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/org.restlet-2.3.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/org.restlet.ext.servlet-2.3.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/presto-parser-0.122.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/protobuf-java-2.5.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/slf4j-log4j12-1.7.7.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/slice-0.10.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/spatial4j-0.6.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/lib/t-digest-3.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/commons-io-2.5.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/httpclient-4.4.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/httpcore-4.4.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/httpmime-4.4.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/jcl-over-slf4j-1.7.7.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/noggit-0.6.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/slf4j-api-1.7.7.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/stax2-api-3.1.4.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/woodstox-core-asl-4.4.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/solrj/lib/zookeeper-3.4.6.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/gmetric4j-1.0.7.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/javax.servlet-api-3.1.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-continuation-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-deploy-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-http-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-io-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-jmx-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-rewrite-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-security-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-server-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-servlet-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-servlets-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-util-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-webapp-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/jetty-xml-9.3.14.v20161028.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/metrics-core-3.1.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/metrics-ganglia-3.1.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/metrics-graphite-3.1.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/metrics-jetty9-3.1.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/server/lib/metrics-jvm-3.1.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/example/example-DIH/solr/db/lib/derby-10.9.1.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/example/example-DIH/solr/db/lib/hsqldb-1.8.0.10.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/core/classes/java:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/test-framework/lib/junit-4.10.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/test-framework/lib/randomizedtesting-runner-2.5.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/antlr-2.7.7.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/antlr4-runtime-4.5.1-1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-core-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-core-api-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-core-avl-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-core-shared-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-i18n-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptor-kerberos-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-admin-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-authn-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-authz-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/api-all-1.0.0-M20.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/bcprov-jdk15-1.45.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/byte-buddy-1.6.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/cglib-nodep-2.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/commons-collections-3.2.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/commons-math3-3.4.1.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/easymock-3.0.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/ehcache-core-2.4.4.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/hadoop-common-2.7.2-tests.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/hadoop-hdfs-2.7.2-tests.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/hadoop-minikdc-2.7.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/jackson-annotations-2.5.4.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/jackson-databind-2.5.4.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/jersey-core-1.9.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/jersey-server-1.9.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/jetty-6.1.26.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/jetty-util-6.1.26.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/mina-core-2.0.0-M5.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/mockito-core-2.6.2.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/netty-all-4.0.36.Final.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/objenesis-2.5.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/presto-parser-0.122.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/test-lib/slice-0.10.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/analysis/icu/lucene-analyzers-icu-6.5.0-SNAPSHOT.jar:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/contrib/solr-analysis-extras/classes/java:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/queryparser/classes/test:/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/contrib/analysis-extras/lib/icu4j-56.1.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-launcher.jar:/export/home/jenkins/.ant/lib/ivy-2.3.0.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-commons-logging.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jai.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-oro.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-swing.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-netrexx.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-bcel.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-regexp.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-bsf.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jsch.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-junit.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-antlr.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-commons-net.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jmf.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jdepend.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-junit4.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-javamail.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-xalan2.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-log4j.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-resolver.jar:/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-testutil.jar:/usr/jdk/instances/jdk1.8.0/lib/tools.jar:/export/home/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.0.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/temp/junit4-J1-20170125_074219_5144731774166579274068.events @/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/temp/junit4-J1-20170125_074219_5142336024307005704168.suites -stdin
   [junit4] ERROR: JVM J1 ended with an exception: java.io.IOException: No space left on device
   [junit4] 	at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
   [junit4] 	at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60)
   [junit4] 	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
   [junit4] 	at sun.nio.ch.IOUtil.write(IOUtil.java:65)
   [junit4] 	at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211)
   [junit4] 	at java.nio.channels.Channels.writeFullyImpl(Channels.java:78)
   [junit4] 	at java.nio.channels.Channels.writeFully(Channels.java:101)
   [junit4] 	at java.nio.channels.Channels.access$000(Channels.java:61)
   [junit4] 	at java.nio.channels.Channels$1.write(Channels.java:174)
   [junit4] 	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
   [junit4] 	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
   [junit4] 	at java.io.FilterOutputStream.close(FilterOutputStream.java:158)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.io.Closer.close(Closer.java:214)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1526)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4] 	at java.lang.Thread.run(Thread.java:745)

BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:775: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:719: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:59: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build.xml:252: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/common-build.xml:536: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/common-build.xml:1444: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/common-build.xml:985: At least one slave process threw an exception, first: No space left on device

Total time: 169 minutes 27 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 632 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/632/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:     1) Thread[id=22825, name=jetty-launcher-3386-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   1) Thread[id=22825, name=jetty-launcher-3386-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
	at __randomizedtesting.SeedInfo.seed([17B30A12BD1077FF]:0)




Build Log:
[...truncated 12424 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_17B30A12BD1077FF-001/init-core-data-001
   [junit4]   2> 2763964 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[17B30A12BD1077FF]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2764008 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[17B30A12BD1077FF]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_17B30A12BD1077FF-001/tempDir-001
   [junit4]   2> 2764009 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[17B30A12BD1077FF]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2764009 INFO  (Thread-7760) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2764009 INFO  (Thread-7760) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2764109 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[17B30A12BD1077FF]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:46804
   [junit4]   2> 2764120 INFO  (jetty-launcher-3386-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2764120 INFO  (jetty-launcher-3386-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2764123 INFO  (jetty-launcher-3386-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@355afc6c{/solr,null,AVAILABLE}
   [junit4]   2> 2764123 INFO  (jetty-launcher-3386-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@bde3538{/solr,null,AVAILABLE}
   [junit4]   2> 2764123 INFO  (jetty-launcher-3386-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4c5c416b{HTTP/1.1,[http/1.1]}{127.0.0.1:51416}
   [junit4]   2> 2764125 INFO  (jetty-launcher-3386-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@a9202d6{HTTP/1.1,[http/1.1]}{127.0.0.1:45075}
   [junit4]   2> 2764125 INFO  (jetty-launcher-3386-thread-1) [    ] o.e.j.s.Server Started @2769272ms
   [junit4]   2> 2764125 INFO  (jetty-launcher-3386-thread-2) [    ] o.e.j.s.Server Started @2769272ms
   [junit4]   2> 2764125 INFO  (jetty-launcher-3386-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45075}
   [junit4]   2> 2764125 INFO  (jetty-launcher-3386-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=51416}
   [junit4]   2> 2764125 ERROR (jetty-launcher-3386-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2764125 ERROR (jetty-launcher-3386-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2764127 INFO  (jetty-launcher-3386-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 2764127 INFO  (jetty-launcher-3386-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2764126 INFO  (jetty-launcher-3386-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 2764127 INFO  (jetty-launcher-3386-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2764127 INFO  (jetty-launcher-3386-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2764127 INFO  (jetty-launcher-3386-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-24T21:57:04.520Z
   [junit4]   2> 2764127 INFO  (jetty-launcher-3386-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2764127 INFO  (jetty-launcher-3386-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-24T21:57:04.520Z
   [junit4]   2> 2764132 INFO  (jetty-launcher-3386-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2764133 INFO  (jetty-launcher-3386-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2764141 INFO  (jetty-launcher-3386-thread-2) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2764141 INFO  (jetty-launcher-3386-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46804/solr
   [junit4]   2> 2764143 INFO  (jetty-launcher-3386-thread-1) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2764144 INFO  (jetty-launcher-3386-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46804/solr
   [junit4]   2> 2764187 INFO  (jetty-launcher-3386-thread-2) [n:127.0.0.1:51416_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:51416_solr
   [junit4]   2> 2764189 INFO  (jetty-launcher-3386-thread-1) [n:127.0.0.1:45075_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45075_solr
   [junit4]   2> 2764189 INFO  (jetty-launcher-3386-thread-2) [n:127.0.0.1:51416_solr    ] o.a.s.c.Overseer Overseer (id=97340294719340549-127.0.0.1:51416_solr-n_0000000000) starting
   [junit4]   2> 2764193 INFO  (zkCallback-3396-thread-1-processing-n:127.0.0.1:51416_solr) [n:127.0.0.1:51416_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2764194 INFO  (zkCallback-3397-thread-1-processing-n:127.0.0.1:45075_solr) [n:127.0.0.1:45075_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2764206 INFO  (jetty-launcher-3386-thread-2) [n:127.0.0.1:51416_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51416_solr
   [junit4]   2> 2764212 INFO  (zkCallback-3397-thread-1-processing-n:127.0.0.1:45075_solr) [n:127.0.0.1:45075_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2764226 INFO  (zkCallback-3396-thread-1-processing-n:127.0.0.1:51416_solr) [n:127.0.0.1:51416_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2764304 INFO  (jetty-launcher-3386-thread-2) [n:127.0.0.1:51416_solr    ] o.a.s.c.CoreContainer Authentication plugin class obtained from system property 'authenticationPlugin': org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 2764305 INFO  (jetty-launcher-3386-thread-2) [n:127.0.0.1:51416_solr    ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=solaris-vm,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
   [junit4]   2> 2764305 INFO  (jetty-launcher-3386-thread-2) [n:127.0.0.1:51416_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 2764308 INFO  (jetty-launcher-3386-thread-2-EventThread) [n:127.0.0.1:51416_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 2764421 INFO  (jetty-launcher-3386-thread-2) [n:127.0.0.1:51416_solr    ] o.a.s.h.c.HttpShardHandlerFactory Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@252a02b4
   [junit4]   2> 2764421 INFO  (jetty-launcher-3386-thread-2) [n:127.0.0.1:51416_solr    ] o.a.s.u.UpdateShardHandler Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@252a02b4
   [junit4]   2> 2764424 INFO  (jetty-launcher-3386-thread-2) [n:127.0.0.1:51416_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_17B30A12BD1077FF-001/tempDir-001/node2/.
   [junit4]   2> 2764625 INFO  (jetty-launcher-3386-thread-1) [n:127.0.0.1:45075_solr    ] o.a.s.c.CoreContainer Authentication plugin class obtained from system property 'authenticationPlugin': org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 2764626 INFO  (jetty-launcher-3386-thread-1) [n:127.0.0.1:45075_solr    ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=solaris-vm,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
   [junit4]   2> 2764626 INFO  (jetty-launcher-3386-thread-1) [n:127.0.0.1:45075_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 2764666 INFO  (jetty-launcher-3386-thread-1-EventThread) [n:127.0.0.1:45075_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 2764875 INFO  (jetty-launcher-3386-thread-1) [n:127.0.0.1:45075_solr    ] o.a.s.h.c.HttpShardHandlerFactory Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@250af9ab
   [junit4]   2> 2764877 INFO  (jetty-launcher-3386-thread-1) [n:127.0.0.1:45075_solr    ] o.a.s.u.UpdateShardHandler Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@250af9ab
   [junit4]   2> 2764881 INFO  (jetty-launcher-3386-thread-1) [n:127.0.0.1:45075_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_17B30A12BD1077FF-001/tempDir-001/node1/.
   [junit4]   2> 2766169 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigHosts
   [junit4]   2> 2766175 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[17B30A12BD1077FF]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noHosts&p=noHosts&t=test&e=1485331026566&s=JI8qH8dfONPgYo8aAMaQL3rhfs0="; Path=/; Domain=127.0.0.1; Expires=Céad, 25-Ean-2017 07:57:06 GMT; HttpOnly". Invalid 'expires' attribute: Céad, 25-Ean-2017 07:57:06 GMT
   [junit4]   2> 2766176 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigHosts
   [junit4]   2> 2766203 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigGroups
   [junit4]   2> 2766210 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[17B30A12BD1077FF]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noGroups&p=noGroups&t=test&e=1485331026598&s=vUJu9uIIOQ33PAEIg2yDHVNa0mk="; Path=/; Domain=127.0.0.1; Expires=Céad, 25-Ean-2017 07:57:06 GMT; HttpOnly". Invalid 'expires' attribute: Céad, 25-Ean-2017 07:57:06 GMT
   [junit4]   2> 2766210 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigGroups
   [junit4]   2> 2766247 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateHost
   [junit4]   2> 2766249 INFO  (qtp768553166-22784) [n:127.0.0.1:51416_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2766249 INFO  (qtp768553166-22784) [n:127.0.0.1:51416_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2766250 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[17B30A12BD1077FF]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=test&e=1485331026642&s=uce+R5UMBA+lSFQPSl2J3NLpUjQ="; Path=/; Domain=127.0.0.1; Expires=Céad, 25-Ean-2017 07:57:06 GMT; HttpOnly". Invalid 'expires' attribute: Céad, 25-Ean-2017 07:57:06 GMT
   [junit4]   2> 2766251 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateHost
   [junit4]   2> 2766290 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNullProxyUser
   [junit4]   2> 2766293 INFO  (qtp768553166-22785) [    ] o.a.s.s.SolrDispatchFilter Error authenticating
   [junit4]   2> java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:220)
   [junit4]   2> 	at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> 	at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:519)
   [junit4]   2> 	at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:128)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:252)
   [junit4]   2> 	at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:248)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:404)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:310)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:296)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:110)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2766293 WARN  (qtp768553166-22785) [    ] o.e.j.s.ServletHandler /solr/admin/collections
   [junit4]   2> org.apache.solr.common.SolrException: Error during request authentication, 
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:410)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:310)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:296)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:110)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:220)
   [junit4]   2> 	at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> 	at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:519)
   [junit4]   2> 	at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:128)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:252)
   [junit4]   2> 	at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:248)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:404)
   [junit4]   2> 	... 26 more
   [junit4]   2> 2766297 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNullProxyUser
   [junit4]   2> 2766335 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidProxyUser
   [junit4]   2> 2766339 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[17B30A12BD1077FF]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bar&p=bar&t=test&e=1485331026730&s=xHEBg/YHBbc2N3tnoXnUa+kip/Y="; Path=/; Domain=127.0.0.1; Expires=Céad, 25-Ean-2017 07:57:06 GMT; HttpOnly". Invalid 'expires' attribute: Céad, 25-Ean-2017 07:57:06 GMT
   [junit4]   2> 2766340 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidProxyUser
   [junit4]   2> 2766382 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testForwarding
   [junit4]   2> 2767279 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[17B30A12BD1077FF]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2767281 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[17B30A12BD1077FF]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:46804/solr ready
   [junit4]   2> 2767283 INFO  (qtp768553166-22780) [n:127.0.0.1:51416_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2767287 INFO  (OverseerThreadFactory-9474-thread-1-processing-n:127.0.0.1:51416_solr) [n:127.0.0.1:51416_solr    ] o.a.s.c.CreateCollectionCmd Create collection forwardingCollection
   [junit4]   2> 2767400 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2
   [junit4]   2> 2767508 INFO  (zkCallback-3397-thread-1-processing-n:127.0.0.1:45075_solr) [n:127.0.0.1:45075_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2768422 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 2768443 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.IndexSchema [forwardingCollection_shard1_replica1] Schema name=test
   [junit4]   2> 2768538 WARN  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.IndexSchema [forwardingCollection_shard1_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2768540 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2768565 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'forwardingCollection_shard1_replica1' using configuration from collection forwardingCollection
   [junit4]   2> 2768566 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SolrCore [[forwardingCollection_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_17B30A12BD1077FF-001/tempDir-001/node1/forwardingCollection_shard1_replica1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_17B30A12BD1077FF-001/tempDir-001/node1/./forwardingCollection_shard1_replica1/data/]
   [junit4]   2> 2768566 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@14d477df
   [junit4]   2> 2768567 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@20e75b4c
   [junit4]   2> 2768616 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2768616 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2768617 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2768617 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2768618 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=27.345703125, floorSegmentMB=0.56640625, forceMergeDeletesPctAllowed=9.652494466806909, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 2768619 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@61eb99ef[forwardingCollection_shard1_replica1] main]
   [junit4]   2> 2768621 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 2768626 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 2768667 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2768668 INFO  (searcherExecutor-9479-thread-1-processing-n:127.0.0.1:45075_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 2768671 INFO  (searcherExecutor-9479-thread-1-processing-n:127.0.0.1:45075_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 2768672 INFO  (searcherExecutor-9479-thread-1-processing-n:127.0.0.1:45075_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 2768672 INFO  (searcherExecutor-9479-thread-1-processing-n:127.0.0.1:45075_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 2768673 INFO  (searcherExecutor-9479-thread-1-processing-n:127.0.0.1:45075_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 2768673 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557444720397910016
   [junit4]   2> 2768674 INFO  (searcherExecutor-9479-thread-1-processing-n:127.0.0.1:45075_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 2768674 INFO  (searcherExecutor-9479-thread-1-processing-n:127.0.0.1:45075_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 2768675 INFO  (searcherExecutor-9479-thread-1-processing-n:127.0.0.1:45075_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 2768675 INFO  (searcherExecutor-9479-thread-1-processing-n:127.0.0.1:45075_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 2768676 INFO  (searcherExecutor-9479-thread-1-processing-n:127.0.0.1:45075_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica1] Registered new searcher Searcher@61eb99ef[forwardingCollection_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2768688 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2768688 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2768688 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45075/solr/forwardingCollection_shard1_replica1/
   [junit4]   2> 2768688 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2768689 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:45075/solr/forwardingCollection_shard1_replica1/ has no replicas
   [junit4]   2> 2768696 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45075/solr/forwardingCollection_shard1_replica1/ shard1
   [junit4]   2> 2768802 INFO  (zkCallback-3397-thread-1-processing-n:127.0.0.1:45075_solr) [n:127.0.0.1:45075_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2768849 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2768856 INFO  (qtp2081386822-22782) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2} status=0 QTime=1456
   [junit4]   2> 2768857 WARN  (httpShardExecutor-9468-thread-1-processing-n:127.0.0.1:51416_solr [http:////127.0.0.1:45075//solr] http:////127.0.0.1:45075//solr) [n:127.0.0.1:51416_solr    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=$&p=$&t=test&e=1485331027792&s=sdnlJwGjB8ZyyapU8mEGGfRV9L8="; Path=/; Domain=127.0.0.1; Expires=Céad, 25-Ean-2017 07:57:07 GMT; HttpOnly". Invalid 'expires' attribute: Céad, 25-Ean-2017 07:57:07 GMT
   [junit4]   2> 2768862 INFO  (qtp768553166-22780) [n:127.0.0.1:51416_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 2768956 INFO  (zkCallback-3397-thread-1-processing-n:127.0.0.1:45075_solr) [n:127.0.0.1:45075_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2769290 INFO  (OverseerCollectionConfigSetProcessor-97340294719340549-127.0.0.1:51416_solr-n_0000000000) [n:127.0.0.1:51416_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2769864 INFO  (qtp768553166-22780) [n:127.0.0.1:51416_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2581
   [junit4]   2> 2769864 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[17B30A12BD1077FF]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=user&p=user&t=test&e=1485331027676&s=pzlUMPE6zXP/mgPIh5eccCnPJrM="; Path=/; Domain=127.0.0.1; Expires=Céad, 25-Ean-2017 07:57:07 GMT; HttpOnly". Invalid 'expires' attribute: Céad, 25-Ean-2017 07:57:07 GMT
   [junit4]   2> 2769865 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[17B30A12BD1077FF]) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: forwardingCollection failOnTimeout:true timeout (sec):100
   [junit4]   2> 2769866 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[17B30A12BD1077FF]) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: forwardingCollection
   [junit4]   2> 2769874 INFO  (qtp2081386822-22778) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.c.S.Request [forwardingCollection_shard1_replica1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 2769874 WARN  (qtp768553166-22785) [n:127.0.0.1:51416_solr    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=$&p=$&t=test&e=1485331030265&s=4M3pzOZTelGVUczNtZMQ25uKPG8="; Path=/; Domain=127.0.0.1; Expires=Céad, 25-Ean-2017 07:57:10 GMT; HttpOnly". Invalid 'expires' attribute: Céad, 25-Ean-2017 07:57:10 GMT
   [junit4]   2> 2769875 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[17B30A12BD1077FF]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=user&p=user&t=test&e=1485331030261&s=UECQgdWShfWq08/E7fi75BL4bMY="; Path=/; Domain=127.0.0.1; Expires=Céad, 25-Ean-2017 07:57:10 GMT; HttpOnly". Invalid 'expires' attribute: Céad, 25-Ean-2017 07:57:10 GMT
   [junit4]   2> 2769875 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[17B30A12BD1077FF]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=$&p=$&t=test&e=1485331030265&s=4M3pzOZTelGVUczNtZMQ25uKPG8="; Path=/; Domain=127.0.0.1; Expires=Céad, 25-Ean-2017 07:57:10 GMT; HttpOnly". Invalid 'expires' attribute: Céad, 25-Ean-2017 07:57:10 GMT
   [junit4]   2> 2769879 INFO  (qtp2081386822-22788) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.c.S.Request [forwardingCollection_shard1_replica1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 2769880 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[17B30A12BD1077FF]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=user&p=user&t=test&e=1485331030272&s=JqNemnkdy1kaYXzWA3AvSfzdoD0="; Path=/; Domain=127.0.0.1; Expires=Céad, 25-Ean-2017 07:57:10 GMT; HttpOnly". Invalid 'expires' attribute: Céad, 25-Ean-2017 07:57:10 GMT
   [junit4]   2> 2769880 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testForwarding
   [junit4]   2> 2769917 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyWrongHost
   [junit4]   2> 2769945 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[17B30A12BD1077FF]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=wrongHost&p=wrongHost&t=test&e=1485331030313&s=i4k6OuiAu4Kdg3s5ilBmPsvVOiQ="; Path=/; Domain=127.0.0.1; Expires=Céad, 25-Ean-2017 07:57:10 GMT; HttpOnly". Invalid 'expires' attribute: Céad, 25-Ean-2017 07:57:10 GMT
   [junit4]   2> 2769946 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyWrongHost
   [junit4]   2> 2769980 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyUnknownRemote
   [junit4]   2> 2780102 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159d27b79140008, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2780103 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159d27b7914000b, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2780103 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159d27b79140007, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2780103 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[17B30A12BD1077FF]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=test&e=1485331030375&s=HUSOiLK0JPcwk6utUo5l+TSreLE="; Path=/; Domain=127.0.0.1; Expires=Céad, 25-Ean-2017 07:57:10 GMT; HttpOnly". Invalid 'expires' attribute: Céad, 25-Ean-2017 07:57:10 GMT
   [junit4]   2> 2780104 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyUnknownRemote
   [junit4]   2> 2780136 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidGroup
   [junit4]   2> 2780159 WARN  (qtp768553166-22780) [n:127.0.0.1:51416_solr    ] o.a.h.s.ShellBasedUnixGroupsMapping got exception trying to get groups for user bar: id: invalid user name: "bar"
   [junit4]   2> 
   [junit4]   2> 2780159 WARN  (qtp768553166-22780) [n:127.0.0.1:51416_solr    ] o.a.h.s.UserGroupInformation No groups available for user bar
   [junit4]   2> 2780161 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[17B30A12BD1077FF]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bogusGroup&p=bogusGroup&t=test&e=1485331040531&s=kRFfWb56O0IjUCS0nYDtW0NQaec="; Path=/; Domain=127.0.0.1; Expires=Céad, 25-Ean-2017 07:57:20 GMT; HttpOnly". Invalid 'expires' attribute: Céad, 25-Ean-2017 07:57:20 GMT
   [junit4]   2> 2780161 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidGroup
   [junit4]   2> 2780190 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateAnyHostAnyUser
   [junit4]   2> 2780192 INFO  (qtp768553166-22784) [n:127.0.0.1:51416_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2780193 INFO  (qtp768553166-22784) [n:127.0.0.1:51416_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2780194 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[17B30A12BD1077FF]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostAnyUser&p=anyHostAnyUser&t=test&e=1485331040585&s=mPQ7atHB5z2FgNp0pDPxD+CC5JQ="; Path=/; Domain=127.0.0.1; Expires=Céad, 25-Ean-2017 07:57:20 GMT; HttpOnly". Invalid 'expires' attribute: Céad, 25-Ean-2017 07:57:20 GMT
   [junit4]   2> 2780194 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateAnyHostAnyUser
   [junit4]   2> 2780203 INFO  (jetty-launcher-3386-thread-2-EventThread) [n:127.0.0.1:51416_solr    ] o.a.c.f.s.ConnectionStateManager State change: SUSPENDED
   [junit4]   2> 2780205 WARN  (zkCallback-3404-thread-2) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1e7f6afa name: ZooKeeperConnection Watcher:127.0.0.1:46804/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 2780205 WARN  (zkCallback-3404-thread-2) [    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 2780229 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidRemote
   [junit4]   2> 2780236 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[17B30A12BD1077FF]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=test&e=1485331040624&s=raQeq9q0g7bhpU/epS0qMcDTTP4="; Path=/; Domain=127.0.0.1; Expires=Céad, 25-Ean-2017 07:57:20 GMT; HttpOnly". Invalid 'expires' attribute: Céad, 25-Ean-2017 07:57:20 GMT
   [junit4]   2> 2780237 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidRemote
   [junit4]   2> 2780271 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateGroup
   [junit4]   2> 2780301 INFO  (qtp768553166-22790) [n:127.0.0.1:51416_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=jenkins&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2780302 INFO  (qtp768553166-22790) [n:127.0.0.1:51416_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=jenkins&action=LIST&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 2780303 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[17B30A12BD1077FF]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostUsersGroup&p=anyHostUsersGroup&t=test&e=1485331040666&s=rGzeKI1NW5yZ8bTdgjnc6+18eUA="; Path=/; Domain=127.0.0.1; Expires=Céad, 25-Ean-2017 07:57:20 GMT; HttpOnly". Invalid 'expires' attribute: Céad, 25-Ean-2017 07:57:20 GMT
   [junit4]   2> 2780303 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[17B30A12BD1077FF]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateGroup
   [junit4]   2> 2780491 INFO  (jetty-launcher-3386-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@4c5c416b{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2780492 INFO  (jetty-launcher-3386-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1795602051
   [junit4]   2> 2780495 INFO  (jetty-launcher-3386-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@a9202d6{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2780496 INFO  (jetty-launcher-3386-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1781360478
   [junit4]   2> 2780498 INFO  (jetty-launcher-3386-thread-1) [    ] o.a.s.c.Overseer Overseer (id=97340294719340549-127.0.0.1:51416_solr-n_0000000000) closing
   [junit4]   2> 2780498 INFO  (OverseerStateUpdate-97340294719340549-127.0.0.1:51416_solr-n_0000000000) [n:127.0.0.1:51416_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:51416_solr
   [junit4]   2> 2780501 WARN  (zkCallback-3396-thread-1-processing-n:127.0.0.1:51416_solr) [n:127.0.0.1:51416_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2780502 INFO  (jetty-launcher-3386-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 2780503 ERROR (Thread[Thread-7777,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:51416_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 2780508 INFO  (zkCallback-3397-thread-2-processing-n:127.0.0.1:45075_solr) [n:127.0.0.1:45075_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 2780509 INFO  (zkCallback-3397-thread-1-processing-n:127.0.0.1:45075_solr) [n:127.0.0.1:45075_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45075_solr
   [junit4]   2> 2780509 INFO  (coreCloseExecutor-9485-thread-1) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@61da8419
   [junit4]   2> 2780512 INFO  (zkCallback-3397-thread-1-processing-n:127.0.0.1:45075_solr) [n:127.0.0.1:45075_solr    ] o.a.s.c.Overseer Overseer (id=97340294719340550-127.0.0.1:45075_solr-n_0000000001) starting
   [junit4]   2> 2780527 INFO  (zkCallback-3397-thread-1-processing-n:127.0.0.1:45075_solr) [n:127.0.0.1:45075_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2780544 INFO  (coreCloseExecutor-9485-thread-1) [n:127.0.0.1:45075_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.forwardingCollection.shard1.replica1
   [junit4]   2> 2780546 INFO  (jetty-launcher-3386-thread-2) [    ] o.a.s.c.Overseer Overseer (id=97340294719340550-127.0.0.1:45075_solr-n_0000000001) closing
   [junit4]   2> 2780547 INFO  (OverseerStateUpdate-97340294719340550-127.0.0.1:45075_solr-n_0000000001) [n:127.0.0.1:45075_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:45075_solr
   [junit4]   2> 2780548 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159d27b79140006, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2781040 INFO  (jetty-launcher-3386-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@355afc6c{/solr,null,UNAVAILABLE}
   [junit4]   2> 2782052 WARN  (zkCallback-3397-thread-1-processing-n:127.0.0.1:45075_solr) [n:127.0.0.1:45075_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2782053 INFO  (jetty-launcher-3386-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 2782053 ERROR (Thread[Thread-7780,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:45075_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 2782174 INFO  (jetty-launcher-3386-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@bde3538{/solr,null,UNAVAILABLE}
   [junit4]   2> 2782178 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[17B30A12BD1077FF]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46804 46804
   [junit4]   2> 2782239 INFO  (Thread-7760) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46804 46804
   [junit4]   2> 2782241 WARN  (Thread-7760) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	11	/solr/overseer/queue
   [junit4]   2> 	8	/solr/overseer/collection-queue-work
   [junit4]   2> 	4	/solr/overseer/queue-work
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	3	/solr/collections
   [junit4]   2> 	3	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 
   [junit4]   2> 2782276 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[17B30A12BD1077FF]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> Jan 24, 2017 9:57:22 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> Jan 24, 2017 9:57:42 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   [junit4]   2>    1) Thread[id=22825, name=jetty-launcher-3386-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2>         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2>         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2>         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2>         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> Jan 24, 2017 9:57:42 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=22825, name=jetty-launcher-3386-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2> 2802469 ERROR (jetty-launcher-3386-thread-2-EventThread) [n:127.0.0.1:51416_solr    ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2> 	at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2> 	at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2> 	at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2> 	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> Jan 24, 2017 9:57:42 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=4, maxDocsPerChunk=414, blockSize=10), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=4, blockSize=10)), sim=RandomSimilarity(queryNorm=true,coord=yes): {}, locale=ga-IE, timezone=America/Santa_Isabel
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=233066712,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [InfoHandlerTest, StatelessScriptUpdateProcessorFactoryTest, JavabinLoaderTest, DisMaxRequestHandlerTest, TestPerFieldSimilarityWithDefaultOverride, TestXmlQParserPlugin, OverseerTest, JSONWriterTest, TestDocumentBuilder, DateFieldTest, DeleteReplicaTest, LeaderElectionTest, TestFunctionQuery, TestUninvertingReader, ReplicationFactorTest, TimeZoneUtilsTest, TestDynamicLoading, TestAnalyzedSuggestions, TestFieldCacheWithThreads, AnalyticsQueryTest, SyncSliceTest, OverriddenZkACLAndCredentialsProvidersTest, BigEndianAscendingWordDeserializerTest, TestFieldCacheSortRandom, AtomicUpdatesTest, ShufflingReplicaListTransformerTest, ManagedSchemaRoundRobinCloudTest, TestRandomDVFaceting, TestUseDocValuesAsStored2, DirectSolrConnectionTest, TestCopyFieldCollectionResource, TestComplexPhraseLeadingWildcard, TestPseudoReturnFields, TestReplicationHandlerBackup, SpatialFilterTest, CdcrRequestHandlerTest, TestConfigOverlay, PrimUtilsTest, TestDownShardTolerantSearch, HdfsUnloadDistributedZkTest, MultiTermTest, SampleTest, TestCloudManagedSchema, TestFieldCache, TestHashQParserPlugin, HdfsRecoverLeaseTest, TestSolrQueryParserResource, TestLazyCores, IndexBasedSpellCheckerTest, HLLUtilTest, HighlighterTest, TriLevelCompositeIdRoutingTest, HdfsCollectionsAPIDistributedZkTest, TestQueryWrapperFilter, TestPKIAuthenticationPlugin, IgnoreCommitOptimizeUpdateProcessorFactoryTest, FileUtilsTest, TestStressRecovery, PrimitiveFieldTypeTest, DistributedExpandComponentTest, TestPerFieldSimilarity, SolrCoreMetricManagerTest, SmileWriterTest, HdfsChaosMonkeyNothingIsSafeTest, PolyFieldTest, SolrTestCaseJ4Test, PKIAuthenticationIntegrationTest, TestCharFilters, UnloadDistributedZkTest, TestPostingsSolrHighlighter, TestRawResponseWriter, OverseerStatusTest, QueryResultKeyTest, XsltUpdateRequestHandlerTest, TestManagedStopFilterFactory, TestBinaryField, TestHdfsCloudBackupRestore, RequestLoggingTest, TestSolrJ, TestCloudInspectUtil, DirectSolrSpellCheckerTest, SuggestComponentContextFilterQueryTest, BufferStoreTest, TestManagedResourceStorage, TestRemoteStreaming, TestRequestForwarding, TestSchemaNameResource, TestFreeTextSuggestions, TestSchemalessBufferedUpdates, TestConfigReload, SpatialRPTFieldTypeTest, TestSolrDynamicMBean, TestHighlightDedupGrouping, SchemaVersionSpecificBehaviorTest, CdcrReplicationDistributedZkTest, QueryElevationComponentTest, TestTolerantUpdateProcessorRandomCloud, ZkStateReaderTest, BadCopyFieldTest, AssignTest, DistributedSpellCheckComponentTest, OutputWriterTest, TestSweetSpotSimilarityFactory, HdfsRestartWhileUpdatingTest, TestGroupingSearch, DirectUpdateHandlerTest, TestFastLRUCache, TestSolrQueryResponse, ResponseLogComponentTest, TestCloudSchemaless, TestCursorMarkWithoutUniqueKey, TestRebalanceLeaders, TestClusterProperties, TestManagedSchemaThreadSafety, DocValuesMultiTest, TestSolrCoreParser, BlockCacheTest, TestMiniSolrCloudClusterSSL, SolrGangliaReporterTest, SolrIndexMetricsTest, TestExactSharedStatsCache, CollectionsAPIAsyncDistributedZkTest, QueryEqualityTest, SystemInfoHandlerTest, SOLR749Test, RulesTest, ConcurrentDeleteAndCreateCollectionTest, TestRandomFlRTGCloud, TestDistributedSearch, TestFieldCollectionResource, TestShortCircuitedRequests, CloneFieldUpdateProcessorFactoryTest, HttpPartitionTest, WrapperMergePolicyFactoryTest, NumericFieldsTest, TestCorePropertiesReload, TestConfigSetProperties, HdfsThreadLeakTest, ZkNodePropsTest, AlternateDirectoryTest, TestCustomDocTransformer, DocValuesMissingTest, TestIntervalFaceting, AnalysisAfterCoreReloadTest, RemoteQueryErrorTest, TestFieldCacheSanityChecker, TestJsonFacets, ConjunctionSolrSpellCheckerTest, TestJmxIntegration, BitVectorTest, TestSchemaSimilarityResource, DistanceUnitsTest, TestSSLRandomization, TestStressLucene, TestManagedSchema, TestCoreContainer, PluginInfoTest, TestRecoveryHdfs, TestTrieFacet, DistributedQueueTest, ConnectionManagerTest, ZkCLITest, OverseerModifyCollectionTest, BaseCdcrDistributedZkTest, CollectionStateFormat2Test, TestAuthenticationFramework, BasicDistributedZkTest, BasicDistributedZk2Test, LeaderElectionIntegrationTest, ShardRoutingTest, BasicZkTest, RecoveryZkTest, TestReplicationHandler, ClusterStateUpdateTest, TestRandomFaceting, ShardRoutingCustomTest, AutoCommitTest, TestFaceting, TestRecovery, TestHashPartitioner, TestRealTimeGet, TestStressReorder, HardAutoCommitTest, TestSolr4Spatial, SpellCheckComponentTest, SolrCmdDistributorTest, PeerSyncTest, BadIndexSchemaTest, BasicFunctionalityTest, TestIndexSearcher, DistributedQueryElevationComponentTest, CurrencyFieldXmlFileTest, SimplePostToolTest, TestCoreDiscovery, SignatureUpdateProcessorFactoryTest, TestExtendedDismaxParser, SuggesterFSTTest, SolrRequestParserTest, TestFoldingMultitermQuery, DocValuesTest, StandardRequestHandlerTest, DocumentAnalysisRequestHandlerTest, TestQueryTypes, TermsComponentTest, DocumentBuilderTest, TermVectorComponentTest, RequiredFieldsTest, IndexSchemaRuntimeFieldTest, TestCollationField, TestCSVResponseWriter, UniqFieldsUpdateProcessorFactoryTest, TestPartialUpdateDeduplication, CSVRequestHandlerTest, TestComponentsName, TestBinaryResponseWriter, TestLFUCache, DOMUtilTest, RAMDirectoryFactoryTest, ClusterStateTest, TestLRUCache, UUIDFieldTest, CircularListTest, DistributedMLTComponentTest, TestRTGBase, CursorPagingTest, TestTolerantSearch, AliasIntegrationTest, CdcrReplicationHandlerTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, CollectionTooManyReplicasTest, DeleteLastCustomShardedReplicaTest, DeleteStatusTest, DistribDocExpirationUpdateProcessorTest, DocValuesNotIndexedTest, LeaderFailoverAfterPartitionTest, LeaderInitiatedRecoveryOnCommitTest, MultiThreadedOCPTest, RecoveryAfterSoftCommitTest, SaslZkACLProviderTest, SharedFSAutoReplicaFailoverTest, SimpleCollectionCreateDeleteTest, SolrCloudExampleTest, TestCollectionAPI, TestCryptoKeys, TestExclusionRuleCollectionAccess, TestLeaderElectionZkExpiry, TestLockTree, TestMiniSolrCloudCluster, TestSolrCloudWithDelegationTokens, TestSolrCloudWithSecureImpersonation]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=17B30A12BD1077FF -Dtests.slow=true -Dtests.locale=ga-IE -Dtests.timezone=America/Santa_Isabel -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J0 | TestSolrCloudWithSecureImpersonation (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   [junit4]    >    1) Thread[id=22825, name=jetty-launcher-3386-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([17B30A12BD1077FF]:0)
   [junit4] Completed [524/681 (1!)] on J0 in 38.52s, 12 tests, 1 error <<< FAILURES!

[...truncated 62994 lines...]