You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2016/12/21 16:22:09 UTC

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1023 - Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1023/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseG1GC

1 tests failed.
FAILED:  org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail

Error Message:
expected:<200> but was:<404>

Stack Trace:
java.lang.AssertionError: expected:<200> but was:<404>
	at __randomizedtesting.SeedInfo.seed([91D9EAAEB180A965:F966DF84611ABB89]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:140)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:304)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11417 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_91D9EAAEB180A965-001/init-core-data-001
   [junit4]   2> 1010319 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[91D9EAAEB180A965]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1010321 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[91D9EAAEB180A965]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_91D9EAAEB180A965-001/tempDir-001
   [junit4]   2> 1010321 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[91D9EAAEB180A965]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1010322 INFO  (Thread-2846) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1010322 INFO  (Thread-2846) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1010422 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[91D9EAAEB180A965]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:52463
   [junit4]   2> 1010430 INFO  (jetty-launcher-9231-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1010430 INFO  (jetty-launcher-9231-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1010433 INFO  (jetty-launcher-9231-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@ceb8096{/solr,null,AVAILABLE}
   [junit4]   2> 1010433 INFO  (jetty-launcher-9231-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7269b6d1{/solr,null,AVAILABLE}
   [junit4]   2> 1010435 INFO  (jetty-launcher-9231-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2a4bc4c6{HTTP/1.1,[http/1.1]}{127.0.0.1:39736}
   [junit4]   2> 1010435 INFO  (jetty-launcher-9231-thread-1) [    ] o.e.j.s.Server Started @1014294ms
   [junit4]   2> 1010435 INFO  (jetty-launcher-9231-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39736}
   [junit4]   2> 1010435 INFO  (jetty-launcher-9231-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@13f51e8{HTTP/1.1,[http/1.1]}{127.0.0.1:41435}
   [junit4]   2> 1010436 INFO  (jetty-launcher-9231-thread-2) [    ] o.e.j.s.Server Started @1014294ms
   [junit4]   2> 1010436 INFO  (jetty-launcher-9231-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41435}
   [junit4]   2> 1010436 ERROR (jetty-launcher-9231-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1010436 ERROR (jetty-launcher-9231-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1010436 INFO  (jetty-launcher-9231-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 1010436 INFO  (jetty-launcher-9231-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 1010436 INFO  (jetty-launcher-9231-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1010436 INFO  (jetty-launcher-9231-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1010436 INFO  (jetty-launcher-9231-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1010436 INFO  (jetty-launcher-9231-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1010436 INFO  (jetty-launcher-9231-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-12-21T15:36:56.184Z
   [junit4]   2> 1010436 INFO  (jetty-launcher-9231-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-12-21T15:36:56.184Z
   [junit4]   2> 1010439 INFO  (jetty-launcher-9231-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1010439 INFO  (jetty-launcher-9231-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1010444 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15922073a070001, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1010447 INFO  (jetty-launcher-9231-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52463/solr
   [junit4]   2> 1010447 INFO  (jetty-launcher-9231-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52463/solr
   [junit4]   2> 1010506 INFO  (jetty-launcher-9231-thread-1) [n:127.0.0.1:39736_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:39736_solr
   [junit4]   2> 1010507 INFO  (jetty-launcher-9231-thread-2) [n:127.0.0.1:41435_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41435_solr
   [junit4]   2> 1010507 INFO  (jetty-launcher-9231-thread-1) [n:127.0.0.1:39736_solr    ] o.a.s.c.Overseer Overseer (id=97146281398829062-127.0.0.1:39736_solr-n_0000000000) starting
   [junit4]   2> 1010518 INFO  (jetty-launcher-9231-thread-1) [n:127.0.0.1:39736_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39736_solr
   [junit4]   2> 1010563 INFO  (jetty-launcher-9231-thread-2) [n:127.0.0.1:41435_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 1010569 INFO  (zkCallback-9242-thread-1-processing-n:127.0.0.1:39736_solr) [n:127.0.0.1:39736_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1010569 INFO  (zkCallback-9241-thread-1-processing-n:127.0.0.1:41435_solr) [n:127.0.0.1:41435_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1010579 INFO  (jetty-launcher-9231-thread-2) [n:127.0.0.1:41435_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
   [junit4]   2> 1010612 INFO  (jetty-launcher-9231-thread-1) [n:127.0.0.1:39736_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 1010612 INFO  (jetty-launcher-9231-thread-1) [n:127.0.0.1:39736_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
   [junit4]   2> 1010666 INFO  (jetty-launcher-9231-thread-1) [n:127.0.0.1:39736_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 1010666 INFO  (jetty-launcher-9231-thread-2) [n:127.0.0.1:41435_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 1010691 INFO  (jetty-launcher-9231-thread-1-EventThread) [n:127.0.0.1:39736_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 1010693 INFO  (jetty-launcher-9231-thread-2-EventThread) [n:127.0.0.1:41435_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 1010829 INFO  (jetty-launcher-9231-thread-1) [n:127.0.0.1:39736_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_91D9EAAEB180A965-001/tempDir-001/node1/.
   [junit4]   2> 1010844 INFO  (jetty-launcher-9231-thread-2) [n:127.0.0.1:41435_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_91D9EAAEB180A965-001/tempDir-001/node2/.
   [junit4]   2> 1011779 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[91D9EAAEB180A965]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationOtherOp
   [junit4]   2> 1011806 INFO  (qtp1550669187-37758) [n:127.0.0.1:39736_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params op=someSolrOperation&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1011806 INFO  (qtp1550669187-37758) [n:127.0.0.1:39736_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={op=someSolrOperation&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1011814 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bar&p=bar&t=test&e=1482370617542&s=qfhMuHj8QwBQGsi0vHFE5Q4xgQk="; Path=/; Domain=127.0.0.1; Expires=gio, 22-dic-2016 01:36:57 GMT; HttpOnly". Invalid 'expires' attribute: gio, 22-dic-2016 01:36:57 GMT
   [junit4]   2> 1011815 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[91D9EAAEB180A965]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationOtherOp
   [junit4]   2> 1011841 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[91D9EAAEB180A965]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancelFail
   [junit4]   2> 1011897 ERROR (qtp375581627-37765) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_1]
   [junit4]   2> 1011899 ERROR (qtp1550669187-37758) [n:127.0.0.1:39736_solr    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_1]
   [junit4]   2> 1011912 ERROR (qtp375581627-37767) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_2]
   [junit4]   2> 1011914 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[91D9EAAEB180A965]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancelFail
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithDelegationTokens -Dtests.method=testDelegationTokenCancelFail -Dtests.seed=91D9EAAEB180A965 -Dtests.slow=true -Dtests.locale=it-CH -Dtests.timezone=Antarctica/McMurdo -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.10s J1 | TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<200> but was:<404>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([91D9EAAEB180A965:F966DF84611ABB89]:0)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:140)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:304)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1011951 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[91D9EAAEB180A965]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenVerify
   [junit4]   2> 1011960 INFO  (qtp1550669187-37763) [n:127.0.0.1:39736_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0BkigFZRhPEZAMEFEm0Y8A28X5az6PzVsn6huKWOdKmB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1011961 INFO  (qtp1550669187-37763) [n:127.0.0.1:39736_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0BkigFZRhPEZAMEFEm0Y8A28X5az6PzVsn6huKWOdKmB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1011963 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1011965 INFO  (qtp375581627-37767) [n:127.0.0.1:41435_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0BkigFZRhPEZAMEFEm0Y8A28X5az6PzVsn6huKWOdKmB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1011965 INFO  (qtp375581627-37767) [n:127.0.0.1:41435_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0BkigFZRhPEZAMEFEm0Y8A28X5az6PzVsn6huKWOdKmB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1011972 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1011974 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52463/solr ready
   [junit4]   2> 1011975 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1011976 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[91D9EAAEB180A965]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenVerify
   [junit4]   2> 1012007 INFO  (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[91D9EAAEB180A965]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testZNodePaths
   [junit4]   2> 1012016 INFO  (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[91D9EAAEB180A965]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testZNodePaths
   [junit4]   2> 1012016 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15922073a07000b, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1012049 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
   [junit4]   2> 1012062 INFO  (qtp1550669187-37758) [n:127.0.0.1:39736_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0DGigFZRhPExgUEFDNbiVtWQnwOAsX4FQXDFJXXvhdyB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1012062 INFO  (qtp1550669187-37758) [n:127.0.0.1:39736_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0DGigFZRhPExgUEFDNbiVtWQnwOAsX4FQXDFJXXvhdyB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1012064 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1012069 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1012070 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52463/solr ready
   [junit4]   2> 1012071 INFO  (qtp1550669187-37758) [n:127.0.0.1:39736_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0DGigFZRhPExgUEFDNbiVtWQnwOAsX4FQXDFJXXvhdyB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1012072 INFO  (qtp1550669187-37758) [n:127.0.0.1:39736_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0DGigFZRhPExgUEFDNbiVtWQnwOAsX4FQXDFJXXvhdyB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1012072 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15922073a07000c, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1012075 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1012091 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1012093 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52463/solr ready
   [junit4]   2> 1012094 INFO  (qtp1550669187-37763) [n:127.0.0.1:39736_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0DgigFZRhPE4AYEFM_UjZTb7fAHUGMdfoWrspUa2btOB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1012095 INFO  (qtp1550669187-37763) [n:127.0.0.1:39736_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0DgigFZRhPE4AYEFM_UjZTb7fAHUGMdfoWrspUa2btOB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1012101 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1012102 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52463/solr ready
   [junit4]   2> 1012103 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1012106 INFO  (qtp375581627-37767) [n:127.0.0.1:41435_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0DgigFZRhPE4AYEFM_UjZTb7fAHUGMdfoWrspUa2btOB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1012106 INFO  (qtp375581627-37767) [n:127.0.0.1:41435_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0DgigFZRhPE4AYEFM_UjZTb7fAHUGMdfoWrspUa2btOB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1012113 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1012114 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52463/solr ready
   [junit4]   2> 1012116 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1012128 INFO  (qtp1550669187-37760) [n:127.0.0.1:39736_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0EKigFZRhPFCgcEFFn3TudMLD_7h2sITRg24RFRhNPXB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1012128 INFO  (qtp1550669187-37760) [n:127.0.0.1:39736_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0EKigFZRhPFCgcEFFn3TudMLD_7h2sITRg24RFRhNPXB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1012130 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1012134 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1012136 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52463/solr ready
   [junit4]   2> 1012137 INFO  (qtp1550669187-37763) [n:127.0.0.1:39736_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0EKigFZRhPFCgcEFFn3TudMLD_7h2sITRg24RFRhNPXB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1012137 INFO  (qtp1550669187-37763) [n:127.0.0.1:39736_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0EKigFZRhPFCgcEFFn3TudMLD_7h2sITRg24RFRhNPXB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1012144 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1012146 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52463/solr ready
   [junit4]   2> 1012147 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1012164 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1012165 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52463/solr ready
   [junit4]   2> 1012167 INFO  (qtp1550669187-37763) [n:127.0.0.1:39736_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0EpigFZRhPFKQgEFL3gXfYjPluQHOf3Jm1vl8XwxhavB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1012167 INFO  (qtp1550669187-37763) [n:127.0.0.1:39736_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0EpigFZRhPFKQgEFL3gXfYjPluQHOf3Jm1vl8XwxhavB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1012173 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1012174 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52463/solr ready
   [junit4]   2> 1012176 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1012182 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1012183 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52463/solr ready
   [junit4]   2> 1012185 INFO  (qtp1550669187-37763) [n:127.0.0.1:39736_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0EpigFZRhPFKQgEFL3gXfYjPluQHOf3Jm1vl8XwxhavB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1012185 INFO  (qtp1550669187-37763) [n:127.0.0.1:39736_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0EpigFZRhPFKQgEFL3gXfYjPluQHOf3Jm1vl8XwxhavB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1012188 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1012189 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[91D9EAAEB180A965]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
   [junit4]   2> 1012217 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[91D9EAAEB180A965]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
   [junit4]   2> 1012224 WARN  (qtp1550669187-37761) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
   [junit4]   2> 1012225 WARN  (qtp1550669187-37761) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 1012225 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1012225 WARN  (qtp375581627-37767) [n:127.0.0.1:41435_solr    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
   [junit4]   2> 1012226 WARN  (qtp375581627-37767) [n:127.0.0.1:41435_solr    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 1012226 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1012232 WARN  (qtp1550669187-37761) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 1012233 WARN  (qtp1550669187-37761) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 1012233 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1012234 WARN  (qtp375581627-37765) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 1012234 WARN  (qtp375581627-37765) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 1012234 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1012234 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[91D9EAAEB180A965]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenewFail
   [junit4]   2> 1012275 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[91D9EAAEB180A965]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenSolrClient
   [junit4]   2> 1012284 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1012287 INFO  (qtp1550669187-37763) [n:127.0.0.1:39736_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0GoigFZRhPFqAsEFOvMegLii8z-gX1nXZOK3wdq9AwpB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1012287 INFO  (qtp1550669187-37763) [n:127.0.0.1:39736_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVkiB0GoigFZRhPFqAsEFOvMegLii8z-gX1nXZOK3wdq9AwpB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1012287 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[91D9EAAEB180A965]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenSolrClient
   [junit4]   2> 1012315 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[91D9EAAEB180A965]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancel
   [junit4]   2> 1012326 ERROR (qtp1550669187-37769) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_12]
   [junit4]   2> 1012335 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1012336 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[91D9EAAEB180A965]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52463/solr ready
   [junit4]   2> 1012338 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1012342 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1012355 ERROR (qtp375581627-37759) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_13]
   [junit4]   2> 1012360 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1012362 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[91D9EAAEB180A965]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=gio, 01-gen-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: gio, 01-gen-1970 00:00:00 GMT
   [junit4]   2> 1012363 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[91D9EAAEB180A965]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancel
   [junit4]   2> 1012365 INFO  (jetty-launcher-9231-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@2a4bc4c6{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1012365 INFO  (jetty-launcher-9231-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@13f51e8{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1012367 INFO  (jetty-launcher-9231-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=120106940
   [junit4]   2> 1012367 INFO  (jetty-launcher-9231-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=2004965628
   [junit4]   2> 1012373 INFO  (jetty-launcher-9231-thread-2) [    ] o.a.s.c.Overseer Overseer (id=97146281398829062-127.0.0.1:39736_solr-n_0000000000) closing
   [junit4]   2> 1012373 INFO  (OverseerStateUpdate-97146281398829062-127.0.0.1:39736_solr-n_0000000000) [n:127.0.0.1:39736_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:39736_solr
   [junit4]   2> 1013878 WARN  (zkCallback-9242-thread-1-processing-n:127.0.0.1:39736_solr) [n:127.0.0.1:39736_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1013878 INFO  (jetty-launcher-9231-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1013879 ERROR (Thread[Thread-2863,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:39736_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 1013880 WARN  (zkCallback-9241-thread-2-processing-n:127.0.0.1:41435_solr) [n:127.0.0.1:41435_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1013880 INFO  (jetty-launcher-9231-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1013880 ERROR (Thread[Thread-2866,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:41435_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 1013890 INFO  (jetty-launcher-9231-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7269b6d1{/solr,null,UNAVAILABLE}
   [junit4]   2> 1013890 INFO  (jetty-launcher-9231-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@ceb8096{/solr,null,UNAVAILABLE}
   [junit4]   2> 1013894 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[91D9EAAEB180A965]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:52463 52463
   [junit4]   2> 1013908 INFO  (Thread-2846) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:52463 52463
   [junit4]   2> 1013909 WARN  (Thread-2846) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	13	/solr/aliases.json
   [junit4]   2> 	13	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	13	/solr/clusterstate.json
   [junit4]   2> 	3	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_7
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_6
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_9
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_8
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_5
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_13
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_12
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_11
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_10
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	13	/solr/live_nodes
   [junit4]   2> 	13	/solr/collections
   [junit4]   2> 	2	/solr/overseer/queue
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 
   [junit4]   2> 1013910 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[91D9EAAEB180A965]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_91D9EAAEB180A965-001
   [junit4]   2> Dec 21, 2016 3:36:59 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 3 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=328, maxMBSortInHeap=6.305109939895468, sim=RandomSimilarity(queryNorm=true): {}, locale=it-CH, timezone=Antarctica/McMurdo
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=1,free=109943192,total=479199232
   [junit4]   2> NOTE: All tests run in this JVM: [ConnectionReuseTest, TestDynamicLoading, BigEndianAscendingWordDeserializerTest, ManagedSchemaRoundRobinCloudTest, TestFuzzyAnalyzedSuggestions, TestDistributedStatsComponentCardinality, TestSimpleTrackingShardHandler, ReturnFieldsTest, DistributedFacetPivotWhiteBoxTest, CoreAdminRequestStatusTest, CSVRequestHandlerTest, TestMiniSolrCloudClusterSSL, WrapperMergePolicyFactoryTest, SpellPossibilityIteratorTest, SuggesterTSTTest, TestCodecSupport, TestLMDirichletSimilarityFactory, UpdateRequestProcessorFactoryTest, ReplicationFactorTest, TestFoldingMultitermQuery, TestSchemaNameResource, TestFieldTypeCollectionResource, HttpSolrCallGetCoreTest, TestNonDefinedSimilarityFactory, BJQParserTest, HdfsUnloadDistributedZkTest, SignatureUpdateProcessorFactoryTest, AnalyticsMergeStrategyTest, TestStandardQParsers, DocExpirationUpdateProcessorFactoryTest, SolrCmdDistributorTest, ConjunctionSolrSpellCheckerTest, ShardRoutingCustomTest, TestHdfsCloudBackupRestore, TestConfigSetsAPIZkFailure, TestSearchPerf, TestRandomCollapseQParserPlugin, DirectUpdateHandlerOptimizeTest, ConfigSetsAPITest, BadComponentTest, TestFilteredDocIdSet, TestQuerySenderListener, TestManagedSchema, TestRTimerTree, BasicAuthIntegrationTest, TestSolrDeletionPolicy2, TestSolrIndexConfig, TestCollapseQParserPlugin, ChangedSchemaMergeTest, TestZkChroot, TestSQLHandlerNonCloud, TestCloudPseudoReturnFields, TestRecovery, TestStressLiveNodes, ShowFileRequestHandlerTest, TestUninvertingReader, TestClassicSimilarityFactory, StatsReloadRaceTest, HdfsRestartWhileUpdatingTest, UpdateParamsTest, HLLUtilTest, TestBulkSchemaAPI, TestTrie, FieldMutatingUpdateProcessorTest, TestCrossCoreJoin, TestSystemIdResolver, BasicFunctionalityTest, HdfsLockFactoryTest, TestPivotHelperCode, TestUnifiedSolrHighlighter, TestBadConfig, TestUseDocValuesAsStored, DeleteInactiveReplicaTest, CurrencyFieldOpenExchangeTest, WordBreakSolrSpellCheckerTest, CdcrBootstrapTest, DistributedFacetPivotSmallAdvancedTest, CollectionsAPISolrJTest, DistribCursorPagingTest, TermVectorComponentTest, DistributedVersionInfoTest, ClassificationUpdateProcessorTest, CollectionReloadTest, CleanupOldIndexTest, TestReplicationHandlerBackup, BasicDistributedZk2Test, TestComponentsName, BadCopyFieldTest, RAMDirectoryFactoryTest, TestDistribIDF, ExitableDirectoryReaderTest, DeleteShardTest, TestInfoStreamLogging, TestSubQueryTransformerDistrib, TestLMJelinekMercerSimilarityFactory, CdcrVersionReplicationTest, TestMacros, TestPHPSerializedResponseWriter, TestFieldCache, AddSchemaFieldsUpdateProcessorFactoryTest, OverseerModifyCollectionTest, CursorMarkTest, SolrIndexSplitterTest, TestScoreJoinQPScore, TestReload, DataDrivenBlockJoinTest, UUIDFieldTest, TestFreeTextSuggestions, SpellCheckComponentTest, SolrCloudExampleTest, TestFiltering, LeaderElectionContextKeyTest, TestOrdValues, URLClassifyProcessorTest, SolrInfoMBeanTest, TestReRankQParserPlugin, TestEmbeddedSolrServerConstructors, TestXmlQParserPlugin, TestPKIAuthenticationPlugin, TestWriterPerf, TestUtils, IndexSchemaTest, ShufflingReplicaListTransformerTest, TestDynamicFieldCollectionResource, AutoCommitTest, TestValueSourceCache, RollingRestartTest, TestRangeQuery, TestRandomFaceting, OverriddenZkACLAndCredentialsProvidersTest, BadIndexSchemaTest, TestQueryTypes, GraphQueryTest, TestLuceneMatchVersion, TestManagedSchemaThreadSafety, TestJoin, DefaultValueUpdateProcessorTest, EchoParamsTest, TestJsonFacets, TestSolrQueryParserResource, HdfsThreadLeakTest, CollectionTooManyReplicasTest, DocValuesMissingTest, NotRequiredUniqueKeyTest, TestSolrCloudWithDelegationTokens]
   [junit4] Completed [261/670 (1!)] on J1 in 4.11s, 8 tests, 1 failure <<< FAILURES!

[...truncated 63750 lines...]