You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/01/28 04:39:53 UTC

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1103 - Still Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1103/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

2 tests failed.
FAILED:  org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail

Error Message:
expected:<200> but was:<404>

Stack Trace:
java.lang.AssertionError: expected:<200> but was:<404>
	at __randomizedtesting.SeedInfo.seed([CFD6B2E0BAB67609:A76987CA6A2C64E5]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:140)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:304)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:     1) Thread[id=19060, name=jetty-launcher-2887-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)    2) Thread[id=19072, name=jetty-launcher-2887-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   1) Thread[id=19060, name=jetty-launcher-2887-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   2) Thread[id=19072, name=jetty-launcher-2887-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
	at __randomizedtesting.SeedInfo.seed([CFD6B2E0BAB67609]:0)




Build Log:
[...truncated 12392 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_CFD6B2E0BAB67609-001/init-core-data-001
   [junit4]   2> 2482683 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[CFD6B2E0BAB67609]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields
   [junit4]   2> 2482686 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[CFD6B2E0BAB67609]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2482687 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[CFD6B2E0BAB67609]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_CFD6B2E0BAB67609-001/tempDir-001
   [junit4]   2> 2482688 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[CFD6B2E0BAB67609]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2482688 INFO  (Thread-5018) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2482688 INFO  (Thread-5018) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2482788 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[CFD6B2E0BAB67609]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:37197
   [junit4]   2> 2482796 INFO  (jetty-launcher-2843-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2482796 INFO  (jetty-launcher-2843-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2482797 INFO  (jetty-launcher-2843-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@d4abfcd{/solr,null,AVAILABLE}
   [junit4]   2> 2482799 INFO  (jetty-launcher-2843-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5259c2b8{HTTP/1.1,[http/1.1]}{127.0.0.1:35673}
   [junit4]   2> 2482799 INFO  (jetty-launcher-2843-thread-1) [    ] o.e.j.s.Server Started @2486576ms
   [junit4]   2> 2482799 INFO  (jetty-launcher-2843-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35673}
   [junit4]   2> 2482800 INFO  (jetty-launcher-2843-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@38bb4e57{/solr,null,AVAILABLE}
   [junit4]   2> 2482800 ERROR (jetty-launcher-2843-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2482800 INFO  (jetty-launcher-2843-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 2482800 INFO  (jetty-launcher-2843-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2482800 INFO  (jetty-launcher-2843-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2482800 INFO  (jetty-launcher-2843-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@75b1cba8{HTTP/1.1,[http/1.1]}{127.0.0.1:39196}
   [junit4]   2> 2482800 INFO  (jetty-launcher-2843-thread-2) [    ] o.e.j.s.Server Started @2486577ms
   [junit4]   2> 2482800 INFO  (jetty-launcher-2843-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39196}
   [junit4]   2> 2482800 INFO  (jetty-launcher-2843-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-28T04:08:02.411Z
   [junit4]   2> 2482800 ERROR (jetty-launcher-2843-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2482801 INFO  (jetty-launcher-2843-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 2482801 INFO  (jetty-launcher-2843-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2482801 INFO  (jetty-launcher-2843-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2482801 INFO  (jetty-launcher-2843-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-28T04:08:02.412Z
   [junit4]   2> 2482804 INFO  (jetty-launcher-2843-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2482804 INFO  (jetty-launcher-2843-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2482812 INFO  (jetty-launcher-2843-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37197/solr
   [junit4]   2> 2482812 INFO  (jetty-launcher-2843-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37197/solr
   [junit4]   2> 2482854 INFO  (jetty-launcher-2843-thread-2) [n:127.0.0.1:39196_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:39196_solr
   [junit4]   2> 2482855 INFO  (jetty-launcher-2843-thread-1) [n:127.0.0.1:35673_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35673_solr
   [junit4]   2> 2482857 INFO  (jetty-launcher-2843-thread-2) [n:127.0.0.1:39196_solr    ] o.a.s.c.Overseer Overseer (id=97358740344143877-127.0.0.1:39196_solr-n_0000000000) starting
   [junit4]   2> 2482859 INFO  (zkCallback-2854-thread-1-processing-n:127.0.0.1:35673_solr) [n:127.0.0.1:35673_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2482859 INFO  (zkCallback-2853-thread-1-processing-n:127.0.0.1:39196_solr) [n:127.0.0.1:39196_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2482870 INFO  (jetty-launcher-2843-thread-2) [n:127.0.0.1:39196_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39196_solr
   [junit4]   2> 2482872 INFO  (zkCallback-2854-thread-1-processing-n:127.0.0.1:35673_solr) [n:127.0.0.1:35673_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2482872 INFO  (zkCallback-2853-thread-1-processing-n:127.0.0.1:39196_solr) [n:127.0.0.1:39196_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2482990 INFO  (jetty-launcher-2843-thread-2) [n:127.0.0.1:39196_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 2482992 INFO  (jetty-launcher-2843-thread-2) [n:127.0.0.1:39196_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
   [junit4]   2> 2482994 INFO  (jetty-launcher-2843-thread-2) [n:127.0.0.1:39196_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 2482999 INFO  (jetty-launcher-2843-thread-2-EventThread) [n:127.0.0.1:39196_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 2483025 INFO  (jetty-launcher-2843-thread-2) [n:127.0.0.1:39196_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_CFD6B2E0BAB67609-001/tempDir-001/node2/.
   [junit4]   2> 2483074 INFO  (jetty-launcher-2843-thread-1) [n:127.0.0.1:35673_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 2483074 INFO  (jetty-launcher-2843-thread-1) [n:127.0.0.1:35673_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
   [junit4]   2> 2483074 INFO  (jetty-launcher-2843-thread-1) [n:127.0.0.1:35673_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 2483092 INFO  (jetty-launcher-2843-thread-1-EventThread) [n:127.0.0.1:35673_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 2483118 INFO  (jetty-launcher-2843-thread-1) [n:127.0.0.1:35673_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_CFD6B2E0BAB67609-001/tempDir-001/node1/.
   [junit4]   2> 2484255 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancelFail
   [junit4]   2> 2484346 ERROR (qtp1434666067-18865) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_1]
   [junit4]   2> 2484348 ERROR (qtp1288311031-18879) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_1]
   [junit4]   2> 2484366 ERROR (qtp1434666067-18868) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_2]
   [junit4]   2> 2484369 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancelFail
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithDelegationTokens -Dtests.method=testDelegationTokenCancelFail -Dtests.seed=CFD6B2E0BAB67609 -Dtests.slow=true -Dtests.locale=en-ZA -Dtests.timezone=Africa/Nairobi -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.14s J1 | TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<200> but was:<404>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([CFD6B2E0BAB67609:A76987CA6A2C64E5]:0)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:140)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:304)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2484409 INFO  (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testZNodePaths
   [junit4]   2> 2484422 INFO  (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testZNodePaths
   [junit4]   2> 2484451 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationOtherOp
   [junit4]   2> 2484457 INFO  (qtp1288311031-18875) [n:127.0.0.1:39196_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params op=someSolrOperation&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2484457 INFO  (qtp1288311031-18875) [n:127.0.0.1:39196_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={op=someSolrOperation&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2484457 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationOtherOp
   [junit4]   2> 2484482 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenVerify
   [junit4]   2> 2484489 INFO  (qtp1288311031-18873) [n:127.0.0.1:39196_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjT9igFaB064_QQCFOdFk_Fuo-YFi3Y1SHd0EDL3ON1KB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2484490 INFO  (qtp1288311031-18873) [n:127.0.0.1:39196_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjT9igFaB064_QQCFOdFk_Fuo-YFi3Y1SHd0EDL3ON1KB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 2484496 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2484497 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37197/solr ready
   [junit4]   2> 2484506 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2484509 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37197/solr ready
   [junit4]   2> 2484511 INFO  (qtp1288311031-18876) [n:127.0.0.1:39196_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjT9igFaB064_QQCFOdFk_Fuo-YFi3Y1SHd0EDL3ON1KB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2484511 INFO  (qtp1288311031-18876) [n:127.0.0.1:39196_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjT9igFaB064_QQCFOdFk_Fuo-YFi3Y1SHd0EDL3ON1KB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2484517 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2484519 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37197/solr ready
   [junit4]   2> 2484521 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenVerify
   [junit4]   2> 2484546 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenSolrClient
   [junit4]   2> 2484559 INFO  (qtp1288311031-18875) [n:127.0.0.1:39196_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjU9igFaB065PQUCFBR0rGcIn7OkdsR3K9btw1Vs89xZB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2484559 INFO  (qtp1288311031-18875) [n:127.0.0.1:39196_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjU9igFaB065PQUCFBR0rGcIn7OkdsR3K9btw1Vs89xZB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2484559 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenSolrClient
   [junit4]   2> 2484603 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
   [junit4]   2> 2484620 INFO  (qtp1288311031-18874) [n:127.0.0.1:39196_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjV2igFaB065dgYCFIa7FEcepjg3vqILDs3vDeBlQ0wPB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2484620 INFO  (qtp1288311031-18874) [n:127.0.0.1:39196_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjV2igFaB065dgYCFIa7FEcepjg3vqILDs3vDeBlQ0wPB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2484627 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2484629 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37197/solr ready
   [junit4]   2> 2484630 INFO  (qtp1288311031-18875) [n:127.0.0.1:39196_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjV2igFaB065dgYCFIa7FEcepjg3vqILDs3vDeBlQ0wPB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2484630 INFO  (qtp1288311031-18875) [n:127.0.0.1:39196_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjV2igFaB065dgYCFIa7FEcepjg3vqILDs3vDeBlQ0wPB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2484652 INFO  (qtp1288311031-18877) [n:127.0.0.1:39196_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjWWigFaB065lgcCFABs5ySKWok8dHR3azgopQ-UDj63B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2484653 INFO  (qtp1288311031-18877) [n:127.0.0.1:39196_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjWWigFaB065lgcCFABs5ySKWok8dHR3azgopQ-UDj63B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2484659 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2484660 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37197/solr ready
   [junit4]   2> 2484665 INFO  (qtp1434666067-18867) [n:127.0.0.1:35673_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjWWigFaB065lgcCFABs5ySKWok8dHR3azgopQ-UDj63B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2484665 INFO  (qtp1434666067-18867) [n:127.0.0.1:35673_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjWWigFaB065lgcCFABs5ySKWok8dHR3azgopQ-UDj63B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2484679 INFO  (qtp1288311031-18874) [n:127.0.0.1:39196_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjW5igFaB065uQgCFIQdmDLCp6WZCD097Eg49CenvsTGB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2484679 INFO  (qtp1288311031-18874) [n:127.0.0.1:39196_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjW5igFaB065uQgCFIQdmDLCp6WZCD097Eg49CenvsTGB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2484684 INFO  (qtp1434666067-18867) [n:127.0.0.1:35673_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjW5igFaB065uQgCFIQdmDLCp6WZCD097Eg49CenvsTGB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2484684 INFO  (qtp1434666067-18867) [n:127.0.0.1:35673_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjW5igFaB065uQgCFIQdmDLCp6WZCD097Eg49CenvsTGB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2484688 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2484690 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37197/solr ready
   [junit4]   2> 2484708 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2484709 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37197/solr ready
   [junit4]   2> 2484711 INFO  (qtp1288311031-18879) [n:127.0.0.1:39196_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjXQigFaB0650AkCFMokvlII97SsSjVv2ApnfX3NuzBPB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2484711 INFO  (qtp1288311031-18879) [n:127.0.0.1:39196_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjXQigFaB0650AkCFMokvlII97SsSjVv2ApnfX3NuzBPB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2484717 INFO  (qtp1434666067-18863) [n:127.0.0.1:35673_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjXQigFaB0650AkCFMokvlII97SsSjVv2ApnfX3NuzBPB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2484717 INFO  (qtp1434666067-18863) [n:127.0.0.1:35673_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnjQjXQigFaB0650AkCFMokvlII97SsSjVv2ApnfX3NuzBPB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2484721 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
   [junit4]   2> 2484748 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
   [junit4]   2> 2484753 WARN  (qtp1288311031-18876) [n:127.0.0.1:39196_solr    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
   [junit4]   2> 2484754 WARN  (qtp1288311031-18876) [n:127.0.0.1:39196_solr    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 2484755 WARN  (qtp1434666067-18871) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
   [junit4]   2> 2484755 WARN  (qtp1434666067-18871) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 2484761 WARN  (qtp1288311031-18874) [n:127.0.0.1:39196_solr    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 2484762 WARN  (qtp1288311031-18874) [n:127.0.0.1:39196_solr    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 2484762 WARN  (qtp1434666067-18865) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 2484763 WARN  (qtp1434666067-18865) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 2484763 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenewFail
   [junit4]   2> 2484794 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancel
   [junit4]   2> 2484810 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2484811 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37197/solr ready
   [junit4]   2> 2484813 ERROR (qtp1288311031-18879) [n:127.0.0.1:39196_solr    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_12]
   [junit4]   2> 2484827 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2484828 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:37197/solr ready
   [junit4]   2> 2484842 ERROR (qtp1434666067-18870) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_13]
   [junit4]   2> 2484851 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancel
   [junit4]   2> 2484853 INFO  (jetty-launcher-2843-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@75b1cba8{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2484854 INFO  (jetty-launcher-2843-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=192804730
   [junit4]   2> 2484854 INFO  (jetty-launcher-2843-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@5259c2b8{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2484855 INFO  (jetty-launcher-2843-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=524452211
   [junit4]   2> 2484859 INFO  (jetty-launcher-2843-thread-1) [    ] o.a.s.c.Overseer Overseer (id=97358740344143877-127.0.0.1:39196_solr-n_0000000000) closing
   [junit4]   2> 2484860 INFO  (OverseerStateUpdate-97358740344143877-127.0.0.1:39196_solr-n_0000000000) [n:127.0.0.1:39196_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:39196_solr
   [junit4]   2> 2484865 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159e3422dfe0005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2486365 WARN  (zkCallback-2853-thread-2-processing-n:127.0.0.1:39196_solr) [n:127.0.0.1:39196_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2486365 WARN  (zkCallback-2854-thread-1-processing-n:127.0.0.1:35673_solr) [n:127.0.0.1:35673_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2486366 INFO  (jetty-launcher-2843-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 2486366 INFO  (jetty-launcher-2843-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 2486366 ERROR (Thread[Thread-5038,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:35673_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 2486366 ERROR (Thread[Thread-5035,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:39196_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 2486376 INFO  (jetty-launcher-2843-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@38bb4e57{/solr,null,UNAVAILABLE}
   [junit4]   2> 2486376 INFO  (jetty-launcher-2843-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@d4abfcd{/solr,null,UNAVAILABLE}
   [junit4]   2> 2486378 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[CFD6B2E0BAB67609]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37197 37197
   [junit4]   2> 2486385 INFO  (Thread-5018) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37197 37197
   [junit4]   2> 2486386 WARN  (Thread-5018) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	11	/solr/aliases.json
   [junit4]   2> 	11	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	11	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_7
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_6
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_9
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_8
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_5
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_12
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_11
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_10
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	11	/solr/live_nodes
   [junit4]   2> 	11	/solr/collections
   [junit4]   2> 	3	/solr/overseer/queue
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 
   [junit4]   2> 2486387 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[CFD6B2E0BAB67609]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_CFD6B2E0BAB67609-001
   [junit4]   2> Jan 28, 2017 4:08:06 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 3 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1647, maxMBSortInHeap=5.597446301178574, sim=RandomSimilarity(queryNorm=true): {}, locale=en-ZA, timezone=Africa/Nairobi
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=176238384,total=530579456
   [junit4]   2> NOTE: All tests run in this JVM: [JSONWriterTest, HdfsBasicDistributedZk2Test, TestSchemaVersionResource, FacetPivotSmallTest, JsonLoaderTest, CleanupOldIndexTest, SolrCoreCheckLockOnStartupTest, TestReversedWildcardFilterFactory, TestSolrCloudWithKerberosAlt, TestConfigOverlay, SolrIndexSplitterTest, TestFilteredDocIdSet, ShowFileRequestHandlerTest, TestFiltering, TestFieldCacheSort, TestSchemalessBufferedUpdates, SSLMigrationTest, DefaultValueUpdateProcessorTest, TestBulkSchemaConcurrent, EchoParamsTest, UpdateRequestProcessorFactoryTest, ChaosMonkeySafeLeaderTest, TestCloudRecovery, TestRuleBasedAuthorizationPlugin, OutOfBoxZkACLAndCredentialsProvidersTest, NotRequiredUniqueKeyTest, DeleteInactiveReplicaTest, SolrCLIZkUtilsTest, TestPushWriter, DistributedFacetPivotLargeTest, RAMDirectoryFactoryTest, TestCryptoKeys, TestSegmentSorting, AddBlockUpdateTest, DeleteShardTest, ResourceLoaderTest, TestSimpleTrackingShardHandler, TestUseDocValuesAsStored2, URLClassifyProcessorTest, TestAnalyzeInfixSuggestions, TestReRankQParserPlugin, TestCoreDiscovery, TestPHPSerializedResponseWriter, HdfsDirectoryTest, DistanceUnitsTest, TestLeaderElectionWithEmptyReplica, HdfsCollectionsAPIDistributedZkTest, SolrCmdDistributorTest, CurrencyFieldOpenExchangeTest, SolrCoreMetricManagerTest, TestInfoStreamLogging, TestSimpleQParserPlugin, TestDFISimilarityFactory, CollectionReloadTest, SpellPossibilityIteratorTest, DistributedFacetPivotSmallAdvancedTest, FileBasedSpellCheckerTest, TestDefaultStatsCache, TestZkChroot, TestRangeQuery, TestJsonFacetRefinement, TestOnReconnectListenerSupport, TestReplicationHandlerBackup, TestJmxMonitoredMap, SharedFSAutoReplicaFailoverUtilsTest, ClusterStateTest, TestCloudDeleteByQuery, TestRecovery, FieldAnalysisRequestHandlerTest, TestSolrCoreProperties, AnalyticsQueryTest, ConfigSetsAPITest, PeerSyncWithIndexFingerprintCachingTest, TestTrie, TestFuzzyAnalyzedSuggestions, TestStressRecovery, StressHdfsTest, CollectionTooManyReplicasTest, JavabinLoaderTest, TestAtomicUpdateErrorCases, ForceLeaderTest, TestQuerySenderListener, TestInPlaceUpdatesStandalone, HighlighterConfigTest, ResponseLogComponentTest, AtomicUpdatesTest, OpenCloseCoreStressTest, NoCacheHeaderTest, TestDefaultSearchFieldResource, ConvertedLegacyTest, EnumFieldTest, TestReloadDeadlock, TestDocSet, ResponseHeaderTest, DeleteNodeTest, TestSolrDynamicMBean, TestHashQParserPlugin, DistributedTermsComponentTest, RestartWhileUpdatingTest, BasicZkTest, DistribCursorPagingTest, TestCrossCoreJoin, TestRestoreCore, SuggesterTest, BlockDirectoryTest, ZkStateReaderTest, HLLUtilTest, TestFieldResource, RequestHandlersTest, MergeStrategyTest, TestFastWriter, CollectionsAPISolrJTest, AliasIntegrationTest, TestUniqueKeyFieldResource, MetricUtilsTest, ReplaceNodeTest, CacheHeaderTest, TestCodecSupport, PreAnalyzedFieldTest, DistributedFacetPivotWhiteBoxTest, TestChildDocTransformer, TestFoldingMultitermQuery, AutoCommitTest, StatsComponentTest, FieldMutatingUpdateProcessorTest, TestManagedSchemaAPI, TestQueryTypes, SortSpecParsingTest, TestSearchPerf, TestLockTree, OverseerCollectionConfigSetProcessorTest, InfoHandlerTest, TestDocBasedVersionConstraints, SolrInfoMBeanTest, ShufflingReplicaListTransformerTest, SolrXmlInZkTest, DistributedIntervalFacetingTest, TestTestInjection, TestComplexPhraseLeadingWildcard, SimpleMLTQParserTest, DistributedQueryComponentOptimizationTest, TestFieldCacheVsDocValues, TestSuggestSpellingConverter, CdcrReplicationHandlerTest, QueryParsingTest, TestExclusionRuleCollectionAccess, TestUnifiedSolrHighlighter, SortByFunctionTest, BooleanFieldTest, TestShardHandlerFactory, TestUtils, TemplateUpdateProcessorTest, TestSchemaResource, TestStressUserVersions, TestFieldCacheReopen, QueryResultKeyTest, CdcrUpdateLogTest, CoreSorterTest, SolrMetricReporterTest, TestFieldTypeResource, SecurityConfHandlerTest, TestSubQueryTransformerCrossCore, DateMathParserTest, TestCustomSort, BasicDistributedZk2Test, CollectionsAPIDistributedZkTest, LeaderElectionIntegrationTest, ShardRoutingTest, FullSolrCloudDistribCmdsTest, TestReplicationHandler, TestRandomFaceting, ZkSolrClientTest, ZkCLITest, TestDistributedGrouping, TestHashPartitioner, TermVectorComponentDistributedTest, TestRealTimeGet, TestStressVersions, SpellCheckComponentTest, TestGroupingSearch, QueryElevationComponentTest, PeerSyncTest, BasicFunctionalityTest, TestBadConfig, SoftAutoCommitTest, SuggesterTSTTest, TestStressLucene, SuggesterWFSTTest, WordBreakSolrSpellCheckerTest, SchemaVersionSpecificBehaviorTest, TestSolrDeletionPolicy1, DebugComponentTest, DisMaxRequestHandlerTest, TestQueryUtils, TestOmitPositions, XmlUpdateRequestHandlerTest, PathHierarchyTokenizerFactoryTest, TestIndexingPerformance, TestSolrQueryParser, IndexSchemaRuntimeFieldTest, RegexBoostProcessorTest, IndexSchemaTest, TestCollationField, ReturnFieldsTest, UniqFieldsUpdateProcessorFactoryTest, CSVRequestHandlerTest, TestLFUCache, SolrTestCaseJ4Test, TestDistributedMissingSort, TestHighlightDedupGrouping, TestTolerantSearch, TestEmbeddedSolrServerConstructors, ConnectionReuseTest, ActionThrottleTest, AssignTest, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrBootstrapTest, ConcurrentDeleteAndCreateCollectionTest, ConnectionManagerTest, CustomCollectionTest, DistributedVersionInfoTest, LeaderFailureAfterFreshStartTest, LeaderInitiatedRecoveryOnShardRestartTest, MissingSegmentRecoveryTest, MultiThreadedOCPTest, OverseerTaskQueueTest, RecoveryAfterSoftCommitTest, RollingRestartTest, ShardSplitTest, TestRequestStatusCollectionAPI, TestShortCircuitedRequests, TestSolrCloudWithDelegationTokens]
   [junit4] Completed [525/684 (1!)] on J1 in 3.98s, 8 tests, 1 failure <<< FAILURES!

[...truncated 7 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_CFD6B2E0BAB67609-001/init-core-data-001
   [junit4]   2> 2486708 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[CFD6B2E0BAB67609]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields
   [junit4]   2> 2486711 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[CFD6B2E0BAB67609]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2486766 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[CFD6B2E0BAB67609]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_CFD6B2E0BAB67609-001/tempDir-001
   [junit4]   2> 2486766 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[CFD6B2E0BAB67609]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2486766 INFO  (Thread-5043) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2486767 INFO  (Thread-5043) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2486866 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[CFD6B2E0BAB67609]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:63899
   [junit4]   2> 2486874 INFO  (jetty-launcher-2887-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2486874 INFO  (jetty-launcher-2887-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2486876 INFO  (jetty-launcher-2887-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@37319379{/solr,null,AVAILABLE}
   [junit4]   2> 2486877 INFO  (jetty-launcher-2887-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@19515245{HTTP/1.1,[http/1.1]}{127.0.0.1:43947}
   [junit4]   2> 2486877 INFO  (jetty-launcher-2887-thread-1) [    ] o.e.j.s.Server Started @2490655ms
   [junit4]   2> 2486877 INFO  (jetty-launcher-2887-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43947}
   [junit4]   2> 2486877 ERROR (jetty-launcher-2887-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2486878 INFO  (jetty-launcher-2887-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 2486878 INFO  (jetty-launcher-2887-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7a72b887{/solr,null,AVAILABLE}
   [junit4]   2> 2486878 INFO  (jetty-launcher-2887-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3d6c38cf{HTTP/1.1,[http/1.1]}{127.0.0.1:34073}
   [junit4]   2> 2486878 INFO  (jetty-launcher-2887-thread-2) [    ] o.e.j.s.Server Started @2490656ms
   [junit4]   2> 2486878 INFO  (jetty-launcher-2887-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34073}
   [junit4]   2> 2486878 ERROR (jetty-launcher-2887-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2486878 INFO  (jetty-launcher-2887-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 2486878 INFO  (jetty-launcher-2887-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2486878 INFO  (jetty-launcher-2887-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2486879 INFO  (jetty-launcher-2887-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-28T04:08:06.490Z
   [junit4]   2> 2486878 INFO  (jetty-launcher-2887-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2486879 INFO  (jetty-launcher-2887-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2486879 INFO  (jetty-launcher-2887-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-28T04:08:06.490Z
   [junit4]   2> 2486881 INFO  (jetty-launcher-2887-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2486882 INFO  (jetty-launcher-2887-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2486888 INFO  (jetty-launcher-2887-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:63899/solr
   [junit4]   2> 2486890 INFO  (jetty-launcher-2887-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:63899/solr
   [junit4]   2> 2486921 INFO  (jetty-launcher-2887-thread-2) [n:127.0.0.1:34073_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:34073_solr
   [junit4]   2> 2486923 INFO  (jetty-launcher-2887-thread-2) [n:127.0.0.1:34073_solr    ] o.a.s.c.Overseer Overseer (id=97358740611334149-127.0.0.1:34073_solr-n_0000000000) starting
   [junit4]   2> 2486931 INFO  (jetty-launcher-2887-thread-1) [n:127.0.0.1:43947_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43947_solr
   [junit4]   2> 2486934 INFO  (zkCallback-2897-thread-1-processing-n:127.0.0.1:34073_solr) [n:127.0.0.1:34073_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2486937 INFO  (jetty-launcher-2887-thread-2) [n:127.0.0.1:34073_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34073_solr
   [junit4]   2> 2486940 INFO  (zkCallback-2897-thread-1-processing-n:127.0.0.1:34073_solr) [n:127.0.0.1:34073_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2486946 INFO  (zkCallback-2898-thread-1-processing-n:127.0.0.1:43947_solr) [n:127.0.0.1:43947_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2486972 INFO  (jetty-launcher-2887-thread-2) [n:127.0.0.1:34073_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 2486972 INFO  (jetty-launcher-2887-thread-2) [n:127.0.0.1:34073_solr    ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=solaris-vm,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
   [junit4]   2> 2486973 INFO  (jetty-launcher-2887-thread-2) [n:127.0.0.1:34073_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 2487010 INFO  (jetty-launcher-2887-thread-2-EventThread) [n:127.0.0.1:34073_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 2487060 INFO  (jetty-launcher-2887-thread-2) [n:127.0.0.1:34073_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_CFD6B2E0BAB67609-001/tempDir-001/node2/.
   [junit4]   2> 2487203 INFO  (jetty-launcher-2887-thread-1) [n:127.0.0.1:43947_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 2487204 INFO  (jetty-launcher-2887-thread-1) [n:127.0.0.1:43947_solr    ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=solaris-vm,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
   [junit4]   2> 2487204 INFO  (jetty-launcher-2887-thread-1) [n:127.0.0.1:43947_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 2487224 INFO  (jetty-launcher-2887-thread-1-EventThread) [n:127.0.0.1:43947_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 2487252 INFO  (jetty-launcher-2887-thread-1) [n:127.0.0.1:43947_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_CFD6B2E0BAB67609-001/tempDir-001/node1/.
   [junit4]   2> 2488204 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidGroup
   [junit4]   2> 2488231 WARN  (qtp1473154163-19024) [    ] o.a.h.s.ShellBasedUnixGroupsMapping got exception trying to get groups for user bar: id: invalid user name: "bar"
   [junit4]   2> 
   [junit4]   2> 2488232 WARN  (qtp1473154163-19024) [    ] o.a.h.s.UserGroupInformation No groups available for user bar
   [junit4]   2> 2488235 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidGroup
   [junit4]   2> 2488261 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigGroups
   [junit4]   2> 2488265 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigGroups
   [junit4]   2> 2488298 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testForwarding
   [junit4]   2> 2489059 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2489061 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:63899/solr ready
   [junit4]   2> 2489064 INFO  (qtp1473154163-19028) [n:127.0.0.1:34073_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2489068 INFO  (OverseerThreadFactory-9165-thread-1-processing-n:127.0.0.1:34073_solr) [n:127.0.0.1:34073_solr    ] o.a.s.c.CreateCollectionCmd Create collection forwardingCollection
   [junit4]   2> 2489178 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2
   [junit4]   2> 2489288 INFO  (zkCallback-2897-thread-2-processing-n:127.0.0.1:34073_solr) [n:127.0.0.1:34073_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2490204 WARN  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 2490205 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 2490231 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.IndexSchema [forwardingCollection_shard1_replica1] Schema name=test
   [junit4]   2> 2490326 WARN  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.IndexSchema [forwardingCollection_shard1_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2490329 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2490348 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'forwardingCollection_shard1_replica1' using configuration from collection forwardingCollection
   [junit4]   2> 2490349 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SolrCore [[forwardingCollection_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_CFD6B2E0BAB67609-001/tempDir-001/node2/forwardingCollection_shard1_replica1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_CFD6B2E0BAB67609-001/tempDir-001/node2/./forwardingCollection_shard1_replica1/data/]
   [junit4]   2> 2490349 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@645deded
   [junit4]   2> 2490351 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=12, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7372609193790265]
   [junit4]   2> 2490387 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2490387 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2490389 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2490389 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2490390 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=37, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 2490390 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@74766b43[forwardingCollection_shard1_replica1] main]
   [junit4]   2> 2490392 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 2490398 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 2490421 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2490421 INFO  (searcherExecutor-9170-thread-1-processing-n:127.0.0.1:34073_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 2490423 INFO  (searcherExecutor-9170-thread-1-processing-n:127.0.0.1:34073_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 2490423 INFO  (searcherExecutor-9170-thread-1-processing-n:127.0.0.1:34073_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 2490423 INFO  (searcherExecutor-9170-thread-1-processing-n:127.0.0.1:34073_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 2490424 INFO  (searcherExecutor-9170-thread-1-processing-n:127.0.0.1:34073_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 2490424 INFO  (searcherExecutor-9170-thread-1-processing-n:127.0.0.1:34073_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 2490424 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557739853614940160
   [junit4]   2> 2490424 INFO  (searcherExecutor-9170-thread-1-processing-n:127.0.0.1:34073_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 2490425 INFO  (searcherExecutor-9170-thread-1-processing-n:127.0.0.1:34073_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 2490425 INFO  (searcherExecutor-9170-thread-1-processing-n:127.0.0.1:34073_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 2490426 INFO  (searcherExecutor-9170-thread-1-processing-n:127.0.0.1:34073_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica1] Registered new searcher Searcher@74766b43[forwardingCollection_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2490437 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2490437 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2490437 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:34073/solr/forwardingCollection_shard1_replica1/
   [junit4]   2> 2490437 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2490438 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:34073/solr/forwardingCollection_shard1_replica1/ has no replicas
   [junit4]   2> 2490444 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:34073/solr/forwardingCollection_shard1_replica1/ shard1
   [junit4]   2> 2490551 INFO  (zkCallback-2897-thread-1-processing-n:127.0.0.1:34073_solr) [n:127.0.0.1:34073_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2490601 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2490605 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2} status=0 QTime=1428
   [junit4]   2> 2490611 INFO  (qtp1473154163-19028) [n:127.0.0.1:34073_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 2490707 INFO  (zkCallback-2897-thread-4-processing-n:127.0.0.1:34073_solr) [n:127.0.0.1:34073_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2491072 INFO  (OverseerCollectionConfigSetProcessor-97358740611334149-127.0.0.1:34073_solr-n_0000000000) [n:127.0.0.1:34073_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2491612 INFO  (qtp1473154163-19028) [n:127.0.0.1:34073_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2548
   [junit4]   2> 2491612 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: forwardingCollection failOnTimeout:true timeout (sec):100
   [junit4]   2> 2491613 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: forwardingCollection
   [junit4]   2> 2491616 INFO  (qtp1473154163-19018) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.c.S.Request [forwardingCollection_shard1_replica1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 2491623 INFO  (qtp1473154163-19027) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.c.S.Request [forwardingCollection_shard1_replica1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 2491623 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testForwarding
   [junit4]   2> 2491648 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigHosts
   [junit4]   2> 2491653 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigHosts
   [junit4]   2> 2491686 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyUnknownRemote
   [junit4]   2> 2511856 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159e3423deb0006, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2511858 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyUnknownRemote
   [junit4]   2> 2511891 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateGroup
   [junit4]   2> 2511936 INFO  (qtp1473154163-19030) [n:127.0.0.1:34073_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=jenkins&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2511937 INFO  (qtp1473154163-19030) [n:127.0.0.1:34073_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=jenkins&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2511937 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateGroup
   [junit4]   2> 2511957 WARN  (zkCallback-2897-thread-4-processing-n:127.0.0.1:34073_solr) [n:127.0.0.1:34073_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3fcfc3c5 name: ZooKeeperConnection Watcher:127.0.0.1:63899/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 2511957 WARN  (zkCallback-2897-thread-4-processing-n:127.0.0.1:34073_solr) [n:127.0.0.1:34073_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 2511958 WARN  (zkCallback-2898-thread-2-processing-n:127.0.0.1:43947_solr) [n:127.0.0.1:43947_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@70815a58 name: ZooKeeperConnection Watcher:127.0.0.1:63899/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 2511960 WARN  (zkCallback-2898-thread-2-processing-n:127.0.0.1:43947_solr) [n:127.0.0.1:43947_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 2511959 WARN  (zkCallback-2905-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@fac215c name: ZooKeeperConnection Watcher:127.0.0.1:63899/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 2511960 WARN  (zkCallback-2905-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 2511963 INFO  (jetty-launcher-2887-thread-1-EventThread) [n:127.0.0.1:43947_solr    ] o.a.c.f.s.ConnectionStateManager State change: SUSPENDED
   [junit4]   2> 2511982 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyWrongHost
   [junit4]   2> 2512001 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyWrongHost
   [junit4]   2> 2512043 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateHost
   [junit4]   2> 2512045 INFO  (qtp1473154163-19026) [n:127.0.0.1:34073_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2512045 INFO  (qtp1473154163-19026) [n:127.0.0.1:34073_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2512045 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateHost
   [junit4]   2> 2512087 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidProxyUser
   [junit4]   2> 2512090 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidProxyUser
   [junit4]   2> 2512132 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidRemote
   [junit4]   2> 2512229 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidRemote
   [junit4]   2> 2512277 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[CFD6B2E0BAB67609]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNullProxyUser
   [junit4]   2> 2512277 INFO  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.SolrDispatchFilter Error authenticating
   [junit4]   2> java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:213)
   [junit4]   2> 	at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> 	at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:519)
   [junit4]   2> 	at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:128)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
   [junit4]   2> 	at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:243)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:404)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:310)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:296)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2512278 WARN  (qtp1473154163-19029) [n:127.0.0.1:34073_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.e.j.s.ServletHandler /solr/admin/collections
   [junit4]   2> org.apache.solr.common.SolrException: Error during request authentication, 
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:410)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:310)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:296)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at o

[...truncated too long message...]

stConfigOverlay, SolrIndexSplitterTest, TestFilteredDocIdSet, ShowFileRequestHandlerTest, TestFiltering, TestFieldCacheSort, TestSchemalessBufferedUpdates, SSLMigrationTest, DefaultValueUpdateProcessorTest, TestBulkSchemaConcurrent, EchoParamsTest, UpdateRequestProcessorFactoryTest, ChaosMonkeySafeLeaderTest, TestCloudRecovery, TestRuleBasedAuthorizationPlugin, OutOfBoxZkACLAndCredentialsProvidersTest, NotRequiredUniqueKeyTest, DeleteInactiveReplicaTest, SolrCLIZkUtilsTest, TestPushWriter, DistributedFacetPivotLargeTest, RAMDirectoryFactoryTest, TestCryptoKeys, TestSegmentSorting, AddBlockUpdateTest, DeleteShardTest, ResourceLoaderTest, TestSimpleTrackingShardHandler, TestUseDocValuesAsStored2, URLClassifyProcessorTest, TestAnalyzeInfixSuggestions, TestReRankQParserPlugin, TestCoreDiscovery, TestPHPSerializedResponseWriter, HdfsDirectoryTest, DistanceUnitsTest, TestLeaderElectionWithEmptyReplica, HdfsCollectionsAPIDistributedZkTest, SolrCmdDistributorTest, CurrencyFieldOpenExchangeTest, SolrCoreMetricManagerTest, TestInfoStreamLogging, TestSimpleQParserPlugin, TestDFISimilarityFactory, CollectionReloadTest, SpellPossibilityIteratorTest, DistributedFacetPivotSmallAdvancedTest, FileBasedSpellCheckerTest, TestDefaultStatsCache, TestZkChroot, TestRangeQuery, TestJsonFacetRefinement, TestOnReconnectListenerSupport, TestReplicationHandlerBackup, TestJmxMonitoredMap, SharedFSAutoReplicaFailoverUtilsTest, ClusterStateTest, TestCloudDeleteByQuery, TestRecovery, FieldAnalysisRequestHandlerTest, TestSolrCoreProperties, AnalyticsQueryTest, ConfigSetsAPITest, PeerSyncWithIndexFingerprintCachingTest, TestTrie, TestFuzzyAnalyzedSuggestions, TestStressRecovery, StressHdfsTest, CollectionTooManyReplicasTest, JavabinLoaderTest, TestAtomicUpdateErrorCases, ForceLeaderTest, TestQuerySenderListener, TestInPlaceUpdatesStandalone, HighlighterConfigTest, ResponseLogComponentTest, AtomicUpdatesTest, OpenCloseCoreStressTest, NoCacheHeaderTest, TestDefaultSearchFieldResource, ConvertedLegacyTest, EnumFieldTest, TestReloadDeadlock, TestDocSet, ResponseHeaderTest, DeleteNodeTest, TestSolrDynamicMBean, TestHashQParserPlugin, DistributedTermsComponentTest, RestartWhileUpdatingTest, BasicZkTest, DistribCursorPagingTest, TestCrossCoreJoin, TestRestoreCore, SuggesterTest, BlockDirectoryTest, ZkStateReaderTest, HLLUtilTest, TestFieldResource, RequestHandlersTest, MergeStrategyTest, TestFastWriter, CollectionsAPISolrJTest, AliasIntegrationTest, TestUniqueKeyFieldResource, MetricUtilsTest, ReplaceNodeTest, CacheHeaderTest, TestCodecSupport, PreAnalyzedFieldTest, DistributedFacetPivotWhiteBoxTest, TestChildDocTransformer, TestFoldingMultitermQuery, AutoCommitTest, StatsComponentTest, FieldMutatingUpdateProcessorTest, TestManagedSchemaAPI, TestQueryTypes, SortSpecParsingTest, TestSearchPerf, TestLockTree, OverseerCollectionConfigSetProcessorTest, InfoHandlerTest, TestDocBasedVersionConstraints, SolrInfoMBeanTest, ShufflingReplicaListTransformerTest, SolrXmlInZkTest, DistributedIntervalFacetingTest, TestTestInjection, TestComplexPhraseLeadingWildcard, SimpleMLTQParserTest, DistributedQueryComponentOptimizationTest, TestFieldCacheVsDocValues, TestSuggestSpellingConverter, CdcrReplicationHandlerTest, QueryParsingTest, TestExclusionRuleCollectionAccess, TestUnifiedSolrHighlighter, SortByFunctionTest, BooleanFieldTest, TestShardHandlerFactory, TestUtils, TemplateUpdateProcessorTest, TestSchemaResource, TestStressUserVersions, TestFieldCacheReopen, QueryResultKeyTest, CdcrUpdateLogTest, CoreSorterTest, SolrMetricReporterTest, TestFieldTypeResource, SecurityConfHandlerTest, TestSubQueryTransformerCrossCore, DateMathParserTest, TestCustomSort, BasicDistributedZk2Test, CollectionsAPIDistributedZkTest, LeaderElectionIntegrationTest, ShardRoutingTest, FullSolrCloudDistribCmdsTest, TestReplicationHandler, TestRandomFaceting, ZkSolrClientTest, ZkCLITest, TestDistributedGrouping, TestHashPartitioner, TermVectorComponentDistributedTest, TestRealTimeGet, TestStressVersions, SpellCheckComponentTest, TestGroupingSearch, QueryElevationComponentTest, PeerSyncTest, BasicFunctionalityTest, TestBadConfig, SoftAutoCommitTest, SuggesterTSTTest, TestStressLucene, SuggesterWFSTTest, WordBreakSolrSpellCheckerTest, SchemaVersionSpecificBehaviorTest, TestSolrDeletionPolicy1, DebugComponentTest, DisMaxRequestHandlerTest, TestQueryUtils, TestOmitPositions, XmlUpdateRequestHandlerTest, PathHierarchyTokenizerFactoryTest, TestIndexingPerformance, TestSolrQueryParser, IndexSchemaRuntimeFieldTest, RegexBoostProcessorTest, IndexSchemaTest, TestCollationField, ReturnFieldsTest, UniqFieldsUpdateProcessorFactoryTest, CSVRequestHandlerTest, TestLFUCache, SolrTestCaseJ4Test, TestDistributedMissingSort, TestHighlightDedupGrouping, TestTolerantSearch, TestEmbeddedSolrServerConstructors, ConnectionReuseTest, ActionThrottleTest, AssignTest, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrBootstrapTest, ConcurrentDeleteAndCreateCollectionTest, ConnectionManagerTest, CustomCollectionTest, DistributedVersionInfoTest, LeaderFailureAfterFreshStartTest, LeaderInitiatedRecoveryOnShardRestartTest, MissingSegmentRecoveryTest, MultiThreadedOCPTest, OverseerTaskQueueTest, RecoveryAfterSoftCommitTest, RollingRestartTest, ShardSplitTest, TestRequestStatusCollectionAPI, TestShortCircuitedRequests, TestSolrCloudWithDelegationTokens, TestSolrCloudWithSecureImpersonation]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=CFD6B2E0BAB67609 -Dtests.slow=true -Dtests.locale=es-PE -Dtests.timezone=Brazil/Acre -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | TestSolrCloudWithSecureImpersonation (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   [junit4]    >    1) Thread[id=19060, name=jetty-launcher-2887-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]    >    2) Thread[id=19072, name=jetty-launcher-2887-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([CFD6B2E0BAB67609]:0)
   [junit4] Completed [528/684 (2!)] on J1 in 47.25s, 12 tests, 1 error <<< FAILURES!

[...truncated 62939 lines...]


[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1113 - Failure!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1113/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseG1GC

3 tests failed.
FAILED:  org.apache.solr.core.TestLazyCores.testNoCommit

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([3EBDB03796DFBC59:E1DD11E65DF8DFFC]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:857)
	at org.apache.solr.core.TestLazyCores.check10(TestLazyCores.java:794)
	at org.apache.solr.core.TestLazyCores.testNoCommit(TestLazyCores.java:776)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound='10']
	xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>

<lst name="responseHeader">
  <int name="status">0</int>
  <int name="QTime">0</int>
  <lst name="params">
    <str name="q">*:*</str>
  </lst>
</lst>
<result name="response" numFound="0" start="0">
</result>
</response>

	request was:q=*:*
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:850)
	... 41 more


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores:     1) Thread[id=12435, name=searcherExecutor-5337-thread-1, state=WAITING, group=TGRP-TestLazyCores]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   1) Thread[id=12435, name=searcherExecutor-5337-thread-1, state=WAITING, group=TGRP-TestLazyCores]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([3EBDB03796DFBC59]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=12435, name=searcherExecutor-5337-thread-1, state=WAITING, group=TGRP-TestLazyCores]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=12435, name=searcherExecutor-5337-thread-1, state=WAITING, group=TGRP-TestLazyCores]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([3EBDB03796DFBC59]:0)




Build Log:
[...truncated 11918 lines...]
   [junit4] Suite: org.apache.solr.core.TestLazyCores
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/init-core-data-001
   [junit4]   2> 1857653 INFO  (SUITE-TestLazyCores-seed#[3EBDB03796DFBC59]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields
   [junit4]   2> 1857655 INFO  (SUITE-TestLazyCores-seed#[3EBDB03796DFBC59]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1857656 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testBadConfigsGenerateErrors
   [junit4]   2> 1857746 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    ] o.a.s.c.CorePropertiesLocator Found 6 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001
   [junit4]   2> 1857746 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    ] o.a.s.c.CorePropertiesLocator Cores are: [core2, core1, badConfig2, badSchema1, badConfig1, badSchema2]
   [junit4]   2> 1857777 INFO  (coreLoadExecutor-5171-thread-3) [    x:badConfig2] o.a.s.c.Config Ignoring exception matching \QSAXParseException\E
   [junit4]   2> 1857825 INFO  (coreLoadExecutor-5171-thread-3) [    x:badSchema1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1857827 INFO  (coreLoadExecutor-5171-thread-1) [    x:core2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1857892 INFO  (coreLoadExecutor-5171-thread-3) [    x:badSchema1] o.a.s.c.Config Ignoring exception matching \QSAXParseException\E
   [junit4]   2> 1857894 INFO  (coreLoadExecutor-5171-thread-1) [    x:core2] o.a.s.s.IndexSchema [core2] Schema name=tiny
   [junit4]   2> 1857894 INFO  (coreLoadExecutor-5171-thread-3) [    x:badConfig1] o.a.s.c.Config Ignoring exception matching \QSAXParseException\E
   [junit4]   2> 1857897 INFO  (coreLoadExecutor-5171-thread-1) [    x:core2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1857898 INFO  (coreLoadExecutor-5171-thread-1) [    x:core2] o.a.s.c.CoreContainer Creating SolrCore 'core2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/core2
   [junit4]   2> 1857899 INFO  (coreLoadExecutor-5171-thread-1) [    x:core2] o.a.s.c.SolrCore [[core2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/core2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/core2/data/]
   [junit4]   2> 1857900 INFO  (coreLoadExecutor-5171-thread-1) [    x:core2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.30431669799336253]
   [junit4]   2> 1857900 INFO  (coreLoadExecutor-5171-thread-3) [    x:badSchema2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1857905 INFO  (coreLoadExecutor-5171-thread-3) [    x:badSchema2] o.a.s.c.Config Ignoring exception matching \QSAXParseException\E
   [junit4]   2> 1857950 INFO  (coreLoadExecutor-5171-thread-1) [    x:core2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1857950 INFO  (coreLoadExecutor-5171-thread-1) [    x:core2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1857951 INFO  (coreLoadExecutor-5171-thread-1) [    x:core2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=40.287109375, floorSegmentMB=0.611328125, forceMergeDeletesPctAllowed=20.8045459139275, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1857951 INFO  (coreLoadExecutor-5171-thread-2) [    x:core1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1857958 INFO  (coreLoadExecutor-5171-thread-2) [    x:core1] o.a.s.s.IndexSchema [core1] Schema name=tiny
   [junit4]   2> 1857960 INFO  (coreLoadExecutor-5171-thread-2) [    x:core1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1857962 INFO  (coreLoadExecutor-5171-thread-2) [    x:core1] o.a.s.c.CoreContainer Creating SolrCore 'core1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/core1
   [junit4]   2> 1857962 INFO  (coreLoadExecutor-5171-thread-2) [    x:core1] o.a.s.c.SolrCore [[core1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/core1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/core1/data/]
   [junit4]   2> 1857963 INFO  (coreLoadExecutor-5171-thread-2) [    x:core1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.30431669799336253]
   [junit4]   2> 1857965 INFO  (coreLoadExecutor-5171-thread-1) [    x:core2] o.a.s.s.SolrIndexSearcher Opening [Searcher@20713e84[core2] main]
   [junit4]   2> 1857966 INFO  (coreLoadExecutor-5171-thread-1) [    x:core2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/core2/conf
   [junit4]   2> 1857966 INFO  (coreLoadExecutor-5171-thread-1) [    x:core2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1857966 INFO  (searcherExecutor-5172-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrCore [core2] Registered new searcher Searcher@20713e84[core2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1858013 INFO  (coreLoadExecutor-5171-thread-2) [    x:core1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1858013 INFO  (coreLoadExecutor-5171-thread-2) [    x:core1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1858013 INFO  (coreLoadExecutor-5171-thread-2) [    x:core1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=40.287109375, floorSegmentMB=0.611328125, forceMergeDeletesPctAllowed=20.8045459139275, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1858014 INFO  (coreLoadExecutor-5171-thread-2) [    x:core1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5efdd719[core1] main]
   [junit4]   2> 1858014 INFO  (coreLoadExecutor-5171-thread-2) [    x:core1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/core1/conf
   [junit4]   2> 1858014 INFO  (coreLoadExecutor-5171-thread-2) [    x:core1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1858015 INFO  (searcherExecutor-5176-thread-1-processing-x:core1) [    x:core1] o.a.s.c.SolrCore [core1] Registered new searcher Searcher@5efdd719[core1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1858018 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1858018 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@32a2a197
   [junit4]   2> 1858042 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@730975ae[core1] main]
   [junit4]   2> 1858042 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1858043 INFO  (searcherExecutor-5176-thread-1) [    ] o.a.s.c.SolrCore [core1] Registered new searcher Searcher@730975ae[core1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):C7)))}
   [junit4]   2> 1858050 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    ] o.a.s.c.S.Request [core1]  webapp=null path=null params={q={!prefix+f%3Dv_t}hel&wt=xml} hits=2 status=0 QTime=7
   [junit4]   2> 1858053 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    ] o.a.s.c.S.Request [core1]  webapp=null path=null params={q={!raw+f%3Dv_t}hello&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 1858054 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    ] o.a.s.c.S.Request [core1]  webapp=null path=null params={q={!raw+f%3Dv_t}Hello&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1858054 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    ] o.a.s.c.S.Request [core1]  webapp=null path=null params={q={!raw+f%3Dv_f}1.5&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1858070 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1858076 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.IndexSchema [badConfig1] Schema name=tiny
   [junit4]   2> 1858077 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1858079 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.CoreContainer Creating SolrCore 'badConfig1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/badConfig1
   [junit4]   2> 1858080 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrCore [[badConfig1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/badConfig1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/badConfig1/data/]
   [junit4]   2> 1858082 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=94.7177734375, floorSegmentMB=2.1044921875, forceMergeDeletesPctAllowed=19.65876337000528, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.25621340687050553
   [junit4]   2> 1858132 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1858132 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1858132 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=96.4296875, floorSegmentMB=0.5283203125, forceMergeDeletesPctAllowed=6.9140874126554355, segmentsPerTier=42.0, maxCFSSegmentSizeMB=1.568359375, noCFSRatio=0.3023830337069574
   [junit4]   2> 1858134 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.SolrIndexSearcher Opening [Searcher@557c50fa[badConfig1] main]
   [junit4]   2> 1858134 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/badConfig1/conf
   [junit4]   2> 1858134 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1858134 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrCores Opening transient core badConfig1
   [junit4]   2> 1858142 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1858148 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.IndexSchema [badConfig2] Schema name=tiny
   [junit4]   2> 1858151 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1858152 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.CoreContainer Creating SolrCore 'badConfig2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/badConfig2
   [junit4]   2> 1858153 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrCore [[badConfig2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/badConfig2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/badConfig2/data/]
   [junit4]   2> 1858155 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1839326344, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1858156 INFO  (searcherExecutor-5186-thread-1-processing-x:badConfig1) [    x:badConfig1] o.a.s.c.SolrCore [badConfig1] Registered new searcher Searcher@557c50fa[badConfig1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1858210 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1858210 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1858210 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=41, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1858229 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.SolrIndexSearcher Opening [Searcher@cdf723c[badConfig2] main]
   [junit4]   2> 1858230 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/badConfig2/conf
   [junit4]   2> 1858230 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1858230 INFO  (searcherExecutor-5190-thread-1-processing-x:badConfig1) [    x:badConfig1] o.a.s.c.SolrCore [badConfig2] Registered new searcher Searcher@cdf723c[badConfig2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1858230 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrCores Opening transient core badConfig2
   [junit4]   2> 1858239 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1858245 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.IndexSchema [badSchema1] Schema name=tiny
   [junit4]   2> 1858247 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1858248 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.CoreContainer Creating SolrCore 'badSchema1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/badSchema1
   [junit4]   2> 1858249 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrCore [[badSchema1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/badSchema1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/badSchema1/data/]
   [junit4]   2> 1858251 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=33.517578125, floorSegmentMB=1.67578125, forceMergeDeletesPctAllowed=12.970342526660431, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3505629858601286
   [junit4]   2> 1858301 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1858301 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1858301 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=99.720703125, floorSegmentMB=1.439453125, forceMergeDeletesPctAllowed=26.804210098491104, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.34522169073305076
   [junit4]   2> 1858302 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.SolrIndexSearcher Opening [Searcher@71956991[badSchema1] main]
   [junit4]   2> 1858302 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/badSchema1/conf
   [junit4]   2> 1858302 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1858303 INFO  (searcherExecutor-5194-thread-1-processing-x:badConfig1) [    x:badConfig1] o.a.s.c.SolrCore [badSchema1] Registered new searcher Searcher@71956991[badSchema1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1858303 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrCores Opening transient core badSchema1
   [junit4]   2> 1858311 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1858317 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.IndexSchema [badSchema2] Schema name=tiny
   [junit4]   2> 1858319 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1858321 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.CoreContainer Creating SolrCore 'badSchema2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/badSchema2
   [junit4]   2> 1858321 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrCore [[badSchema2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/badSchema2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/badSchema2/data/]
   [junit4]   2> 1858323 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=51.3759765625, floorSegmentMB=1.1533203125, forceMergeDeletesPctAllowed=8.228002745005377, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1858375 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1858375 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1858375 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=44.5478515625, floorSegmentMB=1.1611328125, forceMergeDeletesPctAllowed=14.505489697279252, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.42647308619050517
   [junit4]   2> 1858376 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.SolrIndexSearcher Opening [Searcher@192c67ff[badSchema2] main]
   [junit4]   2> 1858377 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-001/badSchema2/conf
   [junit4]   2> 1858377 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1858377 INFO  (searcherExecutor-5198-thread-1-processing-x:badConfig1) [    x:badConfig1] o.a.s.c.SolrCore [badSchema2] Registered new searcher Searcher@192c67ff[badSchema2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1858377 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrCores Opening transient core badSchema2
   [junit4]   2> 1858380 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1858381 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@32a2a197
   [junit4]   2> 1858397 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2581452d[core1] main]
   [junit4]   2> 1858397 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1858398 INFO  (searcherExecutor-5176-thread-1-processing-x:badConfig1) [    x:badConfig1] o.a.s.c.SolrCore [core1] Registered new searcher Searcher@2581452d[core1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.0.0):C7)))}
   [junit4]   2> 1858399 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.S.Request [core1]  webapp=null path=null params={q={!prefix+f%3Dv_t}hel&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 1858400 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.S.Request [core1]  webapp=null path=null params={q={!raw+f%3Dv_t}hello&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 1858400 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.S.Request [core1]  webapp=null path=null params={q={!raw+f%3Dv_t}Hello&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1858400 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.S.Request [core1]  webapp=null path=null params={q={!raw+f%3Dv_f}1.5&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1858401 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1858401 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@476312f9
   [junit4]   2> 1858417 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3db17666[badConfig1] main]
   [junit4]   2> 1858418 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1858418 INFO  (searcherExecutor-5186-thread-1-processing-x:badConfig1) [    x:badConfig1] o.a.s.c.SolrCore [badConfig1] Registered new searcher Searcher@3db17666[badConfig1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):C7)))}
   [junit4]   2> 1858419 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.S.Request [badConfig1]  webapp=null path=null params={q={!prefix+f%3Dv_t}hel&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 1858419 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.S.Request [badConfig1]  webapp=null path=null params={q={!raw+f%3Dv_t}hello&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 1858420 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.S.Request [badConfig1]  webapp=null path=null params={q={!raw+f%3Dv_t}Hello&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1858420 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.S.Request [badConfig1]  webapp=null path=null params={q={!raw+f%3Dv_f}1.5&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1858421 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1858421 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@742b1550
   [junit4]   2> 1858464 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7c08b9a4[badSchema1] main]
   [junit4]   2> 1858464 INFO  (searcherExecutor-5194-thread-1-processing-x:badConfig1) [    x:badConfig1] o.a.s.c.SolrCore [badSchema1] Registered new searcher Searcher@7c08b9a4[badSchema1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):C7)))}
   [junit4]   2> 1858556 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1858556 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.S.Request [badSchema1]  webapp=null path=null params={q={!prefix+f%3Dv_t}hel&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 1858557 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.S.Request [badSchema1]  webapp=null path=null params={q={!raw+f%3Dv_t}hello&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 1858557 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.S.Request [badSchema1]  webapp=null path=null params={q={!raw+f%3Dv_t}Hello&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1858557 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.S.Request [badSchema1]  webapp=null path=null params={q={!raw+f%3Dv_f}1.5&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1858557 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.CoreContainer Shutting down CoreContainer instance=510822852
   [junit4]   2> 1858558 INFO  (coreCloseExecutor-5208-thread-1-processing-x:badConfig1) [    x:core2] o.a.s.c.SolrCore [core2]  CLOSING SolrCore org.apache.solr.core.SolrCore@6bbe4993
   [junit4]   2> 1858558 INFO  (coreCloseExecutor-5208-thread-2-processing-x:badConfig1) [    x:core1] o.a.s.c.SolrCore [core1]  CLOSING SolrCore org.apache.solr.core.SolrCore@3e6ec10c
   [junit4]   2> 1858571 INFO  (coreCloseExecutor-5208-thread-3-processing-x:badConfig1) [    x:badConfig1] o.a.s.c.SolrCore [badConfig1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1733c76e
   [junit4]   2> 1858571 INFO  (coreCloseExecutor-5208-thread-4-processing-x:badConfig1) [    x:badConfig2] o.a.s.c.SolrCore [badConfig2]  CLOSING SolrCore org.apache.solr.core.SolrCore@15218c8a
   [junit4]   2> 1858571 INFO  (coreCloseExecutor-5208-thread-5-processing-x:badConfig1) [    x:badSchema1] o.a.s.c.SolrCore [badSchema1]  CLOSING SolrCore org.apache.solr.core.SolrCore@573405dd
   [junit4]   2> 1858618 INFO  (coreCloseExecutor-5208-thread-6-processing-x:badConfig1) [    x:badSchema2] o.a.s.c.SolrCore [badSchema2]  CLOSING SolrCore org.apache.solr.core.SolrCore@77f19358
   [junit4]   2> 1858618 INFO  (coreCloseExecutor-5208-thread-1-processing-x:badConfig1) [    x:core2] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.core2
   [junit4]   2> 1858635 INFO  (coreCloseExecutor-5208-thread-6-processing-x:badConfig1) [    x:badSchema2] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.badSchema2
   [junit4]   2> 1858635 INFO  (coreCloseExecutor-5208-thread-2-processing-x:badConfig1) [    x:core1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.core1
   [junit4]   2> 1858635 INFO  (coreCloseExecutor-5208-thread-5-processing-x:badConfig1) [    x:badSchema1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.badSchema1
   [junit4]   2> 1858635 INFO  (coreCloseExecutor-5208-thread-4-processing-x:badConfig1) [    x:badConfig2] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.badConfig2
   [junit4]   2> 1858637 INFO  (coreCloseExecutor-5208-thread-3-processing-x:badConfig1) [    x:badConfig1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.badConfig1
   [junit4]   2> 1858638 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1858638 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.SolrTestCaseJ4 ###Ending testBadConfigsGenerateErrors
   [junit4]   2> 1858640 INFO  (TEST-TestLazyCores.testRace-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.SolrTestCaseJ4 ###Starting testRace
   [junit4]   2> 1858662 INFO  (TEST-TestLazyCores.testRace-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrCores Allocating transient cache for 4 transient cores
   [junit4]   2> 1858709 INFO  (coreLoadExecutor-5215-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1858714 INFO  (coreLoadExecutor-5215-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
   [junit4]   2> 1858717 INFO  (coreLoadExecutor-5215-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1858721 INFO  (coreLoadExecutor-5215-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1858723 INFO  (coreLoadExecutor-5215-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
   [junit4]   2> 1858725 INFO  (coreLoadExecutor-5215-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1858726 INFO  (coreLoadExecutor-5215-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
   [junit4]   2> 1858727 INFO  (coreLoadExecutor-5215-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-002/collection1
   [junit4]   2> 1858727 INFO  (coreLoadExecutor-5215-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-002/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-002/collection1/data/]
   [junit4]   2> 1858728 INFO  (coreLoadExecutor-5215-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1858729 INFO  (coreLoadExecutor-5215-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.30431669799336253]
   [junit4]   2> 1858729 INFO  (coreLoadExecutor-5215-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-002/collection2
   [junit4]   2> 1858730 INFO  (coreLoadExecutor-5215-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-002/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-002/collection2/data/]
   [junit4]   2> 1858731 INFO  (coreLoadExecutor-5215-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.30431669799336253]
   [junit4]   2> 1858717 INFO  (coreLoadExecutor-5215-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1858742 INFO  (coreLoadExecutor-5215-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-002/collection5
   [junit4]   2> 1858742 INFO  (coreLoadExecutor-5215-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-002/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-002/collection5/data/]
   [junit4]   2> 1858743 INFO  (coreLoadExecutor-5215-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.30431669799336253]
   [junit4]   2> 1858802 INFO  (coreLoadExecutor-5215-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1858802 INFO  (coreLoadExecutor-5215-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1858802 INFO  (coreLoadExecutor-5215-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=40.287109375, floorSegmentMB=0.611328125, forceMergeDeletesPctAllowed=20.8045459139275, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1858807 INFO  (coreLoadExecutor-5215-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@55c7d682[collection1] main]
   [junit4]   2> 1858808 INFO  (coreLoadExecutor-5215-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-002/collection1/conf
   [junit4]   2> 1858808 INFO  (coreLoadExecutor-5215-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1858814 INFO  (coreLoadExecutor-5215-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1858814 INFO  (coreLoadExecutor-5215-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1858815 INFO  (coreLoadExecutor-5215-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=40.287109375, floorSegmentMB=0.611328125, forceMergeDeletesPctAllowed=20.8045459139275, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1858815 INFO  (searcherExecutor-5216-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@55c7d682[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1858816 INFO  (coreLoadExecutor-5215-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@61d36654[collection2] main]
   [junit4]   2> 1858816 INFO  (coreLoadExecutor-5215-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-002/collection2/conf
   [junit4]   2> 1858816 INFO  (coreLoadExecutor-5215-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1858816 INFO  (coreLoadExecutor-5215-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.c.SolrCores Opening transient core collection2
   [junit4]   2> 1858817 INFO  (searcherExecutor-5217-thread-1-processing-x:collection2) [    x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@61d36654[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1858828 INFO  (coreLoadExecutor-5215-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1858828 INFO  (coreLoadExecutor-5215-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1858829 INFO  (coreLoadExecutor-5215-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=40.287109375, floorSegmentMB=0.611328125, forceMergeDeletesPctAllowed=20.8045459139275, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1858830 INFO  (coreLoadExecutor-5215-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@76d7b51f[collection5] main]
   [junit4]   2> 1858830 INFO  (coreLoadExecutor-5215-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-002/collection5/conf
   [junit4]   2> 1858830 INFO  (coreLoadExecutor-5215-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1858830 INFO  (searcherExecutor-5218-thread-1-processing-x:collection5) [    x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@76d7b51f[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1858842 INFO  (Thread-2844) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1858848 INFO  (Thread-2844) [    x:collection3] o.a.s.s.IndexSchema [collection3] Schema name=tiny
   [junit4]   2> 1858850 INFO  (Thread-2844) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1858851 INFO  (Thread-2844) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection3' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-002/collection3
   [junit4]   2> 1858852 INFO  (Thread-2844) [    x:collection3] o.a.s.c.SolrCore [[collection3] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-002/collection3], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-002/collection3/data/]
   [junit4]   2> 1858853 INFO  (Thread-2844) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.30431669799336253]
   [junit4]   2> 1858900 INFO  (Thread-2844) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1858900 INFO  (Thread-2844) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1858900 INFO  (Thread-2844) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=40.287109375, floorSegmentMB=0.611328125, forceMergeDeletesPctAllowed=20.8045459139275, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1858901 INFO  (Thread-2844) [    x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@643d0dd2[collection3] main]
   [junit4]   2> 1858901 INFO  (Thread-2844) [    x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-002/collection3/conf
   [junit4]   2> 1858902 INFO  (Thread-2844) [    x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1858902 INFO  (searcherExecutor-5228-thread-1-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection3] Registered new searcher Searcher@643d0dd2[collection3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1858902 INFO  (Thread-2844) [    x:collection3] o.a.s.c.SolrCores Opening transient core collection3
   [junit4]   2> 1858907 INFO  (TEST-TestLazyCores.testRace-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.CoreContainer Shutting down CoreContainer instance=482290289
   [junit4]   2> 1858908 INFO  (coreCloseExecutor-5232-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@223fcef7
   [junit4]   2> 1858921 INFO  (coreCloseExecutor-5232-thread-2-processing-x:badConfig1) [    x:collection5] o.a.s.c.SolrCore [collection5]  CLOSING SolrCore org.apache.solr.core.SolrCore@184fde29
   [junit4]   2> 1858934 INFO  (coreCloseExecutor-5232-thread-4-processing-x:badConfig1) [    x:collection3] o.a.s.c.SolrCore [collection3]  CLOSING SolrCore org.apache.solr.core.SolrCore@7901fbf5
   [junit4]   2> 1858935 INFO  (coreCloseExecutor-5232-thread-3-processing-x:badConfig1) [    x:collection2] o.a.s.c.SolrCore [collection2]  CLOSING SolrCore org.apache.solr.core.SolrCore@393ae198
   [junit4]   2> 1858935 INFO  (coreCloseExecutor-5232-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 1858936 INFO  (coreCloseExecutor-5232-thread-2-processing-x:badConfig1) [    x:collection5] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection5
   [junit4]   2> 1858936 INFO  (coreCloseExecutor-5232-thread-4-processing-x:badConfig1) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection3
   [junit4]   2> 1858937 INFO  (coreCloseExecutor-5232-thread-3-processing-x:badConfig1) [    x:collection2] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection2
   [junit4]   2> 1858937 INFO  (TEST-TestLazyCores.testRace-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1858938 INFO  (TEST-TestLazyCores.testRace-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.SolrTestCaseJ4 ###Ending testRace
   [junit4]   2> 1858940 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.SolrTestCaseJ4 ###Starting testCachingLimit
   [junit4]   2> 1858961 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrCores Allocating transient cache for 4 transient cores
   [junit4]   2> 1859017 INFO  (coreLoadExecutor-5239-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1859018 INFO  (coreLoadExecutor-5239-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1859024 INFO  (coreLoadExecutor-5239-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
   [junit4]   2> 1859026 INFO  (coreLoadExecutor-5239-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1859027 INFO  (coreLoadExecutor-5239-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
   [junit4]   2> 1859028 INFO  (coreLoadExecutor-5239-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection5
   [junit4]   2> 1859028 INFO  (coreLoadExecutor-5239-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection5/data/]
   [junit4]   2> 1859029 INFO  (coreLoadExecutor-5239-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1859030 INFO  (coreLoadExecutor-5239-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.30431669799336253]
   [junit4]   2> 1859030 INFO  (coreLoadExecutor-5239-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection2
   [junit4]   2> 1859030 INFO  (coreLoadExecutor-5239-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection2/data/]
   [junit4]   2> 1859022 INFO  (coreLoadExecutor-5239-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1859037 INFO  (coreLoadExecutor-5239-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
   [junit4]   2> 1859039 INFO  (coreLoadExecutor-5239-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1859040 INFO  (coreLoadExecutor-5239-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.30431669799336253]
   [junit4]   2> 1859054 INFO  (coreLoadExecutor-5239-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection1
   [junit4]   2> 1859054 INFO  (coreLoadExecutor-5239-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection1/data/]
   [junit4]   2> 1859056 INFO  (coreLoadExecutor-5239-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.30431669799336253]
   [junit4]   2> 1859109 INFO  (coreLoadExecutor-5239-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1859109 INFO  (coreLoadExecutor-5239-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1859109 INFO  (coreLoadExecutor-5239-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1859109 INFO  (coreLoadExecutor-5239-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1859110 INFO  (coreLoadExecutor-5239-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=40.287109375, floorSegmentMB=0.611328125, forceMergeDeletesPctAllowed=20.8045459139275, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1859110 INFO  (coreLoadExecutor-5239-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=40.287109375, floorSegmentMB=0.611328125, forceMergeDeletesPctAllowed=20.8045459139275, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1859111 INFO  (coreLoadExecutor-5239-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@4d1ee421[collection2] main]
   [junit4]   2> 1859111 INFO  (coreLoadExecutor-5239-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@459cfcb6[collection1] main]
   [junit4]   2> 1859111 INFO  (coreLoadExecutor-5239-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection2/conf
   [junit4]   2> 1859111 INFO  (coreLoadExecutor-5239-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection1/conf
   [junit4]   2> 1859111 INFO  (coreLoadExecutor-5239-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1859111 INFO  (coreLoadExecutor-5239-thread-1-processing-x:badConfig1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1859112 INFO  (coreLoadExecutor-5239-thread-2-processing-x:badConfig1) [    x:collection2] o.a.s.c.SolrCores Opening transient core collection2
   [junit4]   2> 1859112 INFO  (searcherExecutor-5242-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@459cfcb6[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1859112 INFO  (searcherExecutor-5241-thread-1-processing-x:collection2) [    x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@4d1ee421[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1859115 INFO  (coreLoadExecutor-5239-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1859115 INFO  (coreLoadExecutor-5239-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1859115 INFO  (coreLoadExecutor-5239-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=40.287109375, floorSegmentMB=0.611328125, forceMergeDeletesPctAllowed=20.8045459139275, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1859116 INFO  (coreLoadExecutor-5239-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@40b22b6c[collection5] main]
   [junit4]   2> 1859116 INFO  (coreLoadExecutor-5239-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection5/conf
   [junit4]   2> 1859117 INFO  (coreLoadExecutor-5239-thread-3-processing-x:badConfig1) [    x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1859117 INFO  (searcherExecutor-5240-thread-1-processing-x:collection5) [    x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@40b22b6c[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1859126 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1859132 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.IndexSchema [collection3] Schema name=tiny
   [junit4]   2> 1859134 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1859136 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.CoreContainer Creating SolrCore 'collection3' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection3
   [junit4]   2> 1859136 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrCore [[collection3] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection3], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection3/data/]
   [junit4]   2> 1859138 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1439233372, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1859189 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1859189 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1859189 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=72.009765625, floorSegmentMB=1.7744140625, forceMergeDeletesPctAllowed=10.523669280573735, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1275017424424136
   [junit4]   2> 1859190 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6b55bbd7[collection3] main]
   [junit4]   2> 1859191 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection3/conf
   [junit4]   2> 1859191 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1859191 INFO  (searcherExecutor-5252-thread-1-processing-x:badConfig1) [    x:badConfig1] o.a.s.c.SolrCore [collection3] Registered new searcher Searcher@6b55bbd7[collection3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1859191 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrCores Opening transient core collection3
   [junit4]   2> 1859200 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1859206 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.IndexSchema [collection4] Schema name=tiny
   [junit4]   2> 1859208 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1859209 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.CoreContainer Creating SolrCore 'collection4' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection4
   [junit4]   2> 1859210 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrCore [[collection4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection4/data/]
   [junit4]   2> 1859212 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=17, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1859262 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1859262 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1859262 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=64.2119140625, floorSegmentMB=0.333984375, forceMergeDeletesPctAllowed=1.5232896522589834, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1592136936880964
   [junit4]   2> 1859263 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3d29fd93[collection4] main]
   [junit4]   2> 1859264 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection4/conf
   [junit4]   2> 1859264 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1859264 INFO  (searcherExecutor-5256-thread-1-processing-x:badConfig1) [    x:badConfig1] o.a.s.c.SolrCore [collection4] Registered new searcher Searcher@3d29fd93[collection4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1859273 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1859278 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.IndexSchema [collection6] Schema name=tiny
   [junit4]   2> 1859281 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1859283 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.CoreContainer Creating SolrCore 'collection6' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection6
   [junit4]   2> 1859283 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrCore [[collection6] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection6], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection6/data/]
   [junit4]   2> 1859286 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=47, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1859336 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1859336 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1859337 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=0.0595703125, floorSegmentMB=0.35546875, forceMergeDeletesPctAllowed=17.647140057379904, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1859337 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.SolrIndexSearcher Opening [Searcher@483476a6[collection6] main]
   [junit4]   2> 1859338 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection6/conf
   [junit4]   2> 1859338 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1859338 INFO  (searcherExecutor-5260-thread-1-processing-x:badConfig1) [    x:badConfig1] o.a.s.c.SolrCore [collection6] Registered new searcher Searcher@483476a6[collection6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1859338 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrCores Opening transient core collection6
   [junit4]   2> 1859347 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1859353 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.IndexSchema [collection7] Schema name=tiny
   [junit4]   2> 1859355 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1859357 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.CoreContainer Creating SolrCore 'collection7' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection7
   [junit4]   2> 1859358 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrCore [[collection7] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection7], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection7/data/]
   [junit4]   2> 1859359 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=27, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1859413 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1859413 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1859414 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=64.2412109375, floorSegmentMB=1.65234375, forceMergeDeletesPctAllowed=20.818059834297582, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1859415 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3271807a[collection7] main]
   [junit4]   2> 1859415 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection7/conf
   [junit4]   2> 1859415 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1859416 INFO  (searcherExecutor-5264-thread-1-processing-x:badConfig1) [    x:badConfig1] o.a.s.c.SolrCore [collection7] Registered new searcher Searcher@3271807a[collection7] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1859416 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrCores Opening transient core collection7
   [junit4]   2> 1859425 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1859431 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.IndexSchema [collection8] Schema name=tiny
   [junit4]   2> 1859433 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1859435 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.CoreContainer Creating SolrCore 'collection8' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection8
   [junit4]   2> 1859435 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.c.SolrCore [[collection8] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection8], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection8/data/]
   [junit4]   2> 1859437 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=46.8935546875, floorSegmentMB=0.9716796875, forceMergeDeletesPctAllowed=21.36750255821817, segmentsPerTier=43.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1996304254213528
   [junit4]   2> 1859492 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1859492 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1859492 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=48324511, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1859493 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1d485dd[collection8] main]
   [junit4]   2> 1859493 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_3EBDB03796DFBC59-001/tempDir-003/collection8/conf
   [junit4]   2> 1859494 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1859494 INFO  (searcherExecutor-5268-thread-1-processing-x:badConfig1) [    x:badConfig1] o.a.s.c.SolrCore [collection8] Registered new searcher Searcher@1d485dd[collection8] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1859494 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[3EBDB03796DFBC59]) [    x:badConfig1] o.a

[...truncated too long message...]

ava.lang.Thread.run(Thread.java:745)
   [junit4]   2> Feb 02, 2017 1:03:45 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=12435, name=searcherExecutor-5337-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2> Feb 02, 2017 1:03:48 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=12435, name=searcherExecutor-5337-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {v_t=FSTOrd50, id=FSTOrd50, v_s=PostingsFormat(name=Direct), v_f=PostingsFormat(name=Memory)}, docValues:{}, maxPointsInLeafNode=355, maxMBSortInHeap=7.995895916396819, sim=RandomSimilarity(queryNorm=false): {}, locale=sl, timezone=Etc/GMT+1
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=2,free=106857856,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [OpenExchangeRatesOrgProviderTest, HdfsRecoverLeaseTest, ZkControllerTest, DateRangeFieldTest, TestOverriddenPrefixQueryForCustomFieldType, SearchHandlerTest, TestSolrCloudWithHadoopAuthPlugin, BasicDistributedZkTest, TestSolrQueryParserDefaultOperatorResource, TestPostingsSolrHighlighter, TestUniqueKeyFieldResource, LeaderFailureAfterFreshStartTest, TestMacroExpander, CleanupOldIndexTest, SparseHLLTest, TestHighlightDedupGrouping, SampleTest, AnalyticsQueryTest, TestFieldCacheSort, SimpleMLTQParserTest, DocumentBuilderTest, TestSchemalessBufferedUpdates, DistribCursorPagingTest, BinaryUpdateRequestHandlerTest, TestLFUCache, SharedFSAutoReplicaFailoverUtilsTest, PolyFieldTest, TestFileDictionaryLookup, TestUseDocValuesAsStored2, TestOnReconnectListenerSupport, TestCollectionAPIs, TestRequestStatusCollectionAPI, ZkCLITest, TestIBSimilarityFactory, DirectSolrConnectionTest, TestRestManager, TestConfigReload, ZkNodePropsTest, TestFaceting, StandardRequestHandlerTest, TestTolerantSearch, VersionInfoTest, RestartWhileUpdatingTest, TestBM25SimilarityFactory, ShufflingReplicaListTransformerTest, IndexSchemaRuntimeFieldTest, TestObjectReleaseTracker, BooleanFieldTest, CollectionTooManyReplicasTest, TestReloadDeadlock, TestSolrCloudWithKerberosAlt, LeaderElectionContextKeyTest, OverseerTest, TestScoreJoinQPNoScore, DocValuesTest, TestStressReorder, TestRestoreCore, TestHighFrequencyDictionaryFactory, DateFieldTest, TestCloudDeleteByQuery, OutputWriterTest, TestLegacyNumericRangeQueryBuilder, StatsReloadRaceTest, TestDistributedStatsComponentCardinality, V2ApiIntegrationTest, ConnectionReuseTest, SolrPluginUtilsTest, SuggesterFSTTest, TestSort, TestSubQueryTransformerCrossCore, TestMissingGroups, DistributedFacetPivotWhiteBoxTest, DistributedExpandComponentTest, TestSweetSpotSimilarityFactory, DistributedMLTComponentTest, TestCloudRecovery, TestAnalyzeInfixSuggestions, TestSegmentSorting, TestRandomCollapseQParserPlugin, TestInPlaceUpdatesStandalone, TestDFISimilarityFactory, TolerantUpdateProcessorTest, TestChildDocTransformer, TestSolrFieldCacheMBean, TestCloudPivotFacet, ResponseHeaderTest, UniqFieldsUpdateProcessorFactoryTest, TestDistribDocBasedVersion, HdfsCollectionsAPIDistributedZkTest, QueryEqualityTest, SimplePostToolTest, TestXmlQParser, TestSizeLimitedDistributedMap, SolrCoreTest, TestFieldResource, DistributedFacetPivotSmallAdvancedTest, TestRandomDVFaceting, ShardRoutingTest, TestSolrDeletionPolicy1, TestBinaryField, TestDefaultSearchFieldResource, FastVectorHighlighterTest, PreAnalyzedUpdateProcessorTest, TestShardHandlerFactory, TestDocTermOrds, TestSolrCloudSnapshots, CoreAdminRequestStatusTest, TestRawTransformer, CoreSorterTest, SolrTestCaseJ4Test, SpatialFilterTest, TestCoreAdminApis, TestCSVResponseWriter, TestSchemaResource, TestStressUserVersions, TestStressVersions, TestSolrCLIRunExample, InfoHandlerTest, PeerSyncTest, TermsComponentTest, SuggestComponentContextFilterQueryTest, ForceLeaderTest, HdfsThreadLeakTest, TestSolr4Spatial2, RecoveryAfterSoftCommitTest, TestPartialUpdateDeduplication, TestPivotHelperCode, EnumFieldTest, RecoveryZkTest, CdcrReplicationHandlerTest, TestConfigSetsAPIExclusivity, TestDocumentBuilder, TestUpdate, TestExactSharedStatsCache, TestCollationField, DisMaxRequestHandlerTest, TestHashQParserPlugin, TestFieldCacheReopen, TestFunctionQuery, LeaderInitiatedRecoveryOnShardRestartTest, PeerSyncWithIndexFingerprintCachingTest, TestLRUStatsCache, ClusterStateUpdateTest, TestRuleBasedAuthorizationPlugin, DeleteNodeTest, TestLRUCache, TestPseudoReturnFields, HLLSerializationTest, RequestLoggingTest, TestDocBasedVersionConstraints, DistribJoinFromCollectionTest, TestHashPartitioner, HdfsBasicDistributedZk2Test, IgnoreCommitOptimizeUpdateProcessorFactoryTest, BlockJoinFacetDistribTest, SchemaVersionSpecificBehaviorTest, TriLevelCompositeIdRoutingTest, TestManagedResource, CdcrVersionReplicationTest, TestGroupingSearch, TestAuthorizationFramework, LeaderElectionTest, TestReRankQParserPlugin, TestExtendedDismaxParser, UtilsToolTest, TestStressLucene, TestXIncludeConfig, TestLeaderInitiatedRecoveryThread, CollectionsAPIDistributedZkTest, LeaderElectionIntegrationTest, TestRandomFaceting, TestDistributedSearch, SolrCmdDistributorTest, TestLazyCores]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLazyCores -Dtests.seed=3EBDB03796DFBC59 -Dtests.slow=true -Dtests.locale=sl -Dtests.timezone=Etc/GMT+1 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J0 | TestLazyCores (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   [junit4]    >    1) Thread[id=12435, name=searcherExecutor-5337-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([3EBDB03796DFBC59]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=12435, name=searcherExecutor-5337-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([3EBDB03796DFBC59]:0)
   [junit4] Completed [384/692 (1!)] on J0 in 33.58s, 9 tests, 3 errors <<< FAILURES!

[...truncated 988 lines...]
   [junit4] JVM J1: stdout was not empty, see: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/temp/junit4-J1-20170202_123214_1233712632626860448083.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/heapdumps/java_pid25220.hprof ...
   [junit4] Heap dump file created [124724171 bytes in 1.104 secs]
   [junit4] <<< JVM J1: EOF ----

[...truncated 11650 lines...]
BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:775: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:727: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid25220.hprof

Total time: 62 minutes 53 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1112 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1112/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC

3 tests failed.
FAILED:  org.apache.solr.core.TestLazyCores.testNoCommit

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([EBC9B0A6EFBEE1A3:34A9117724998206]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:857)
	at org.apache.solr.core.TestLazyCores.check10(TestLazyCores.java:794)
	at org.apache.solr.core.TestLazyCores.testNoCommit(TestLazyCores.java:776)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound='10']
	xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>

<lst name="responseHeader">
  <int name="status">0</int>
  <int name="QTime">0</int>
  <lst name="params">
    <str name="q">*:*</str>
  </lst>
</lst>
<result name="response" numFound="0" start="0">
</result>
</response>

	request was:q=*:*
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:850)
	... 41 more


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores:     1) Thread[id=21305, name=searcherExecutor-6430-thread-1, state=WAITING, group=TGRP-TestLazyCores]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   1) Thread[id=21305, name=searcherExecutor-6430-thread-1, state=WAITING, group=TGRP-TestLazyCores]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([EBC9B0A6EFBEE1A3]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=21305, name=searcherExecutor-6430-thread-1, state=WAITING, group=TGRP-TestLazyCores]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=21305, name=searcherExecutor-6430-thread-1, state=WAITING, group=TGRP-TestLazyCores]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([EBC9B0A6EFBEE1A3]:0)




Build Log:
[...truncated 11795 lines...]
   [junit4] Suite: org.apache.solr.core.TestLazyCores
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/init-core-data-001
   [junit4]   2> 1957351 INFO  (SUITE-TestLazyCores-seed#[EBC9B0A6EFBEE1A3]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields
   [junit4]   2> 1957353 INFO  (SUITE-TestLazyCores-seed#[EBC9B0A6EFBEE1A3]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1957357 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testLazyLoad
   [junit4]   2> 1957378 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    ] o.a.s.c.SolrCores Allocating transient cache for 4 transient cores
   [junit4]   2> 1957428 INFO  (coreLoadExecutor-6369-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1957431 INFO  (coreLoadExecutor-6369-thread-2) [    x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1957431 INFO  (coreLoadExecutor-6369-thread-3) [    x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1957439 INFO  (coreLoadExecutor-6369-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
   [junit4]   2> 1957439 INFO  (coreLoadExecutor-6369-thread-2) [    x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
   [junit4]   2> 1957440 INFO  (coreLoadExecutor-6369-thread-3) [    x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
   [junit4]   2> 1957443 INFO  (coreLoadExecutor-6369-thread-2) [    x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1957443 INFO  (coreLoadExecutor-6369-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1957443 INFO  (coreLoadExecutor-6369-thread-3) [    x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1957445 INFO  (coreLoadExecutor-6369-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection1
   [junit4]   2> 1957445 INFO  (coreLoadExecutor-6369-thread-3) [    x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection5
   [junit4]   2> 1957445 INFO  (coreLoadExecutor-6369-thread-2) [    x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection2
   [junit4]   2> 1957445 INFO  (coreLoadExecutor-6369-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection1/data/]
   [junit4]   2> 1957445 INFO  (coreLoadExecutor-6369-thread-3) [    x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection5/data/]
   [junit4]   2> 1957445 INFO  (coreLoadExecutor-6369-thread-2) [    x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection2/data/]
   [junit4]   2> 1957447 INFO  (coreLoadExecutor-6369-thread-2) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1957447 INFO  (coreLoadExecutor-6369-thread-3) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1957447 INFO  (coreLoadExecutor-6369-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1957522 INFO  (coreLoadExecutor-6369-thread-3) [    x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1957522 INFO  (coreLoadExecutor-6369-thread-3) [    x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1957522 INFO  (coreLoadExecutor-6369-thread-3) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=33.572265625, floorSegmentMB=0.2099609375, forceMergeDeletesPctAllowed=1.6390811190138121, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1957523 INFO  (coreLoadExecutor-6369-thread-3) [    x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@1b014dbf[collection5] main]
   [junit4]   2> 1957524 INFO  (coreLoadExecutor-6369-thread-3) [    x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection5/conf
   [junit4]   2> 1957524 INFO  (coreLoadExecutor-6369-thread-3) [    x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1957524 INFO  (coreLoadExecutor-6369-thread-2) [    x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1957524 INFO  (coreLoadExecutor-6369-thread-2) [    x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1957524 INFO  (searcherExecutor-6371-thread-1-processing-x:collection5) [    x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@1b014dbf[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1957524 INFO  (coreLoadExecutor-6369-thread-2) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=33.572265625, floorSegmentMB=0.2099609375, forceMergeDeletesPctAllowed=1.6390811190138121, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1957525 INFO  (coreLoadExecutor-6369-thread-2) [    x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@1b2dd799[collection2] main]
   [junit4]   2> 1957525 INFO  (coreLoadExecutor-6369-thread-2) [    x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection2/conf
   [junit4]   2> 1957526 INFO  (coreLoadExecutor-6369-thread-2) [    x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1957526 INFO  (coreLoadExecutor-6369-thread-2) [    x:collection2] o.a.s.c.SolrCores Opening transient core collection2
   [junit4]   2> 1957526 INFO  (searcherExecutor-6372-thread-1-processing-x:collection2) [    x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@1b2dd799[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1957540 INFO  (coreLoadExecutor-6369-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1957540 INFO  (coreLoadExecutor-6369-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1957540 INFO  (coreLoadExecutor-6369-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=33.572265625, floorSegmentMB=0.2099609375, forceMergeDeletesPctAllowed=1.6390811190138121, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1957541 INFO  (coreLoadExecutor-6369-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6bb66695[collection1] main]
   [junit4]   2> 1957541 INFO  (coreLoadExecutor-6369-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection1/conf
   [junit4]   2> 1957542 INFO  (coreLoadExecutor-6369-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1957542 INFO  (searcherExecutor-6370-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6bb66695[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1957552 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1957559 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.IndexSchema [collection3] Schema name=tiny
   [junit4]   2> 1957562 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1957564 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection3' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection3
   [junit4]   2> 1957564 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCore [[collection3] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection3], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection3/data/]
   [junit4]   2> 1957566 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=32.9833984375, floorSegmentMB=1.064453125, forceMergeDeletesPctAllowed=11.094967552806862, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.22001476198417516
   [junit4]   2> 1957615 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1957615 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1957615 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=97.66796875, floorSegmentMB=1.443359375, forceMergeDeletesPctAllowed=28.355999409807378, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5530456735746854
   [junit4]   2> 1957616 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@232354b0[collection3] main]
   [junit4]   2> 1957617 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection3/conf
   [junit4]   2> 1957617 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1957617 INFO  (searcherExecutor-6382-thread-1-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection3] Registered new searcher Searcher@232354b0[collection3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1957617 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCores Opening transient core collection3
   [junit4]   2> 1957626 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1957633 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.IndexSchema [collection4] Schema name=tiny
   [junit4]   2> 1957636 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1957638 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection4' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection4
   [junit4]   2> 1957638 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCore [[collection4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection4/data/]
   [junit4]   2> 1957640 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1957694 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1957695 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1957695 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=24, maxMergedSegmentMB=82.939453125, floorSegmentMB=0.2421875, forceMergeDeletesPctAllowed=0.909469125898934, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1957696 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@5ce6ec05[collection4] main]
   [junit4]   2> 1957696 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-001/collection4/conf
   [junit4]   2> 1957696 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1957696 INFO  (searcherExecutor-6386-thread-1-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection4] Registered new searcher Searcher@5ce6ec05[collection4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1957697 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.CoreContainer Shutting down CoreContainer instance=939392129
   [junit4]   2> 1957697 INFO  (coreCloseExecutor-6390-thread-1-processing-x:collection3) [    x:collection5] o.a.s.c.SolrCore [collection5]  CLOSING SolrCore org.apache.solr.core.SolrCore@4de84703
   [junit4]   2> 1957697 INFO  (coreCloseExecutor-6390-thread-2-processing-x:collection3) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1ebf6384
   [junit4]   2> 1957697 INFO  (coreCloseExecutor-6390-thread-3-processing-x:collection3) [    x:collection4] o.a.s.c.SolrCore [collection4]  CLOSING SolrCore org.apache.solr.core.SolrCore@36ee08da
   [junit4]   2> 1957698 INFO  (coreCloseExecutor-6390-thread-4-processing-x:collection3) [    x:collection2] o.a.s.c.SolrCore [collection2]  CLOSING SolrCore org.apache.solr.core.SolrCore@656ad8f7
   [junit4]   2> 1957698 INFO  (coreCloseExecutor-6390-thread-2-processing-x:collection3) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 1957699 INFO  (coreCloseExecutor-6390-thread-5-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection3]  CLOSING SolrCore org.apache.solr.core.SolrCore@6253e26c
   [junit4]   2> 1957700 INFO  (coreCloseExecutor-6390-thread-3-processing-x:collection3) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection4
   [junit4]   2> 1957700 INFO  (coreCloseExecutor-6390-thread-1-processing-x:collection3) [    x:collection5] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection5
   [junit4]   2> 1957701 INFO  (coreCloseExecutor-6390-thread-4-processing-x:collection3) [    x:collection2] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection2
   [junit4]   2> 1957701 INFO  (coreCloseExecutor-6390-thread-5-processing-x:collection3) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection3
   [junit4]   2> 1957701 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1957701 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.SolrTestCaseJ4 ###Ending testLazyLoad
   [junit4]   2> 1957705 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.SolrTestCaseJ4 ###Starting testNoCommit
   [junit4]   2> 1957728 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCores Allocating transient cache for 4 transient cores
   [junit4]   2> 1957768 INFO  (coreLoadExecutor-6397-thread-1-processing-x:collection3) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1957768 INFO  (coreLoadExecutor-6397-thread-3-processing-x:collection3) [    x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1957768 INFO  (coreLoadExecutor-6397-thread-2-processing-x:collection3) [    x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1957778 INFO  (coreLoadExecutor-6397-thread-3-processing-x:collection3) [    x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
   [junit4]   2> 1957778 INFO  (coreLoadExecutor-6397-thread-1-processing-x:collection3) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
   [junit4]   2> 1957778 INFO  (coreLoadExecutor-6397-thread-2-processing-x:collection3) [    x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
   [junit4]   2> 1957790 INFO  (coreLoadExecutor-6397-thread-2-processing-x:collection3) [    x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1957790 INFO  (coreLoadExecutor-6397-thread-1-processing-x:collection3) [    x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1957790 INFO  (coreLoadExecutor-6397-thread-3-processing-x:collection3) [    x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1957792 INFO  (coreLoadExecutor-6397-thread-2-processing-x:collection3) [    x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection2
   [junit4]   2> 1957792 INFO  (coreLoadExecutor-6397-thread-1-processing-x:collection3) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection1
   [junit4]   2> 1957792 INFO  (coreLoadExecutor-6397-thread-3-processing-x:collection3) [    x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection5
   [junit4]   2> 1957793 INFO  (coreLoadExecutor-6397-thread-2-processing-x:collection3) [    x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection2/data/]
   [junit4]   2> 1957793 INFO  (coreLoadExecutor-6397-thread-1-processing-x:collection3) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection1/data/]
   [junit4]   2> 1957793 INFO  (coreLoadExecutor-6397-thread-3-processing-x:collection3) [    x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection5/data/]
   [junit4]   2> 1957794 INFO  (coreLoadExecutor-6397-thread-2-processing-x:collection3) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1957794 INFO  (coreLoadExecutor-6397-thread-1-processing-x:collection3) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1957796 INFO  (coreLoadExecutor-6397-thread-3-processing-x:collection3) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1957868 INFO  (coreLoadExecutor-6397-thread-2-processing-x:collection3) [    x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1957868 INFO  (coreLoadExecutor-6397-thread-2-processing-x:collection3) [    x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1957869 INFO  (coreLoadExecutor-6397-thread-2-processing-x:collection3) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=33.572265625, floorSegmentMB=0.2099609375, forceMergeDeletesPctAllowed=1.6390811190138121, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1957870 INFO  (coreLoadExecutor-6397-thread-2-processing-x:collection3) [    x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@1e54d456[collection2] main]
   [junit4]   2> 1957870 INFO  (coreLoadExecutor-6397-thread-3-processing-x:collection3) [    x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1957870 INFO  (coreLoadExecutor-6397-thread-2-processing-x:collection3) [    x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection2/conf
   [junit4]   2> 1957870 INFO  (coreLoadExecutor-6397-thread-3-processing-x:collection3) [    x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1957871 INFO  (coreLoadExecutor-6397-thread-3-processing-x:collection3) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=33.572265625, floorSegmentMB=0.2099609375, forceMergeDeletesPctAllowed=1.6390811190138121, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1957871 INFO  (coreLoadExecutor-6397-thread-2-processing-x:collection3) [    x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1957872 INFO  (searcherExecutor-6398-thread-1-processing-x:collection2) [    x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@1e54d456[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1957872 INFO  (coreLoadExecutor-6397-thread-2-processing-x:collection3) [    x:collection2] o.a.s.c.SolrCores Opening transient core collection2
   [junit4]   2> 1957872 INFO  (coreLoadExecutor-6397-thread-3-processing-x:collection3) [    x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@29270c4c[collection5] main]
   [junit4]   2> 1957872 INFO  (coreLoadExecutor-6397-thread-3-processing-x:collection3) [    x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection5/conf
   [junit4]   2> 1957873 INFO  (coreLoadExecutor-6397-thread-3-processing-x:collection3) [    x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1957873 INFO  (searcherExecutor-6400-thread-1-processing-x:collection5) [    x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@29270c4c[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1957875 INFO  (coreLoadExecutor-6397-thread-1-processing-x:collection3) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1957875 INFO  (coreLoadExecutor-6397-thread-1-processing-x:collection3) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1957876 INFO  (coreLoadExecutor-6397-thread-1-processing-x:collection3) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=33.572265625, floorSegmentMB=0.2099609375, forceMergeDeletesPctAllowed=1.6390811190138121, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1957876 INFO  (coreLoadExecutor-6397-thread-1-processing-x:collection3) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@cb4dac1[collection1] main]
   [junit4]   2> 1957876 INFO  (coreLoadExecutor-6397-thread-1-processing-x:collection3) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection1/conf
   [junit4]   2> 1957877 INFO  (coreLoadExecutor-6397-thread-1-processing-x:collection3) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1957877 INFO  (searcherExecutor-6399-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@cb4dac1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1957887 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1957893 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.IndexSchema [collection3] Schema name=tiny
   [junit4]   2> 1957896 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1957898 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection3' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection3
   [junit4]   2> 1957898 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCore [[collection3] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection3], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection3/data/]
   [junit4]   2> 1957900 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=38, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1957951 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1957951 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1957951 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1879721739, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1957952 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@55f9bd9a[collection3] main]
   [junit4]   2> 1957952 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection3/conf
   [junit4]   2> 1957953 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1957953 INFO  (searcherExecutor-6410-thread-1-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection3] Registered new searcher Searcher@55f9bd9a[collection3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1957953 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCores Opening transient core collection3
   [junit4]   2> 1957962 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1957977 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.IndexSchema [collection6] Schema name=tiny
   [junit4]   2> 1957980 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1957981 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection6' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection6
   [junit4]   2> 1957982 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCore [[collection6] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection6], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection6/data/]
   [junit4]   2> 1957983 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10853435200115502]
   [junit4]   2> 1958030 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1958030 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1958031 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=12, maxMergedSegmentMB=94.447265625, floorSegmentMB=1.9111328125, forceMergeDeletesPctAllowed=12.08612530104242, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24250091677968424
   [junit4]   2> 1958031 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@78aaf388[collection6] main]
   [junit4]   2> 1958032 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection6/conf
   [junit4]   2> 1958032 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1958032 INFO  (searcherExecutor-6414-thread-1-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection6] Registered new searcher Searcher@78aaf388[collection6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1958032 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCores Opening transient core collection6
   [junit4]   2> 1958038 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1958047 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.IndexSchema [collection7] Schema name=tiny
   [junit4]   2> 1958050 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1958051 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection7' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection7
   [junit4]   2> 1958052 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCore [[collection7] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection7], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection7/data/]
   [junit4]   2> 1958054 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7221227726719222]
   [junit4]   2> 1958101 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1958101 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1958101 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=78.6513671875, floorSegmentMB=1.40625, forceMergeDeletesPctAllowed=23.512416323001037, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6829357045737356
   [junit4]   2> 1958102 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@74d95514[collection7] main]
   [junit4]   2> 1958102 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection7/conf
   [junit4]   2> 1958102 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1958102 INFO  (searcherExecutor-6418-thread-1-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection7] Registered new searcher Searcher@74d95514[collection7] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1958103 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCores Opening transient core collection7
   [junit4]   2> 1958112 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1958119 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.IndexSchema [collection8] Schema name=tiny
   [junit4]   2> 1958130 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1958132 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection8' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection8
   [junit4]   2> 1958132 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCore [[collection8] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection8], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection8/data/]
   [junit4]   2> 1958134 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1830386754, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1958183 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1958183 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1958184 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=66.345703125, floorSegmentMB=0.61328125, forceMergeDeletesPctAllowed=0.6257725298446926, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6957615097359832
   [junit4]   2> 1958184 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@28e9ea27[collection8] main]
   [junit4]   2> 1958185 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection8/conf
   [junit4]   2> 1958185 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1958185 INFO  (searcherExecutor-6422-thread-1-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection8] Registered new searcher Searcher@28e9ea27[collection8] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1958185 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCores Opening transient core collection8
   [junit4]   2> 1958185 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCores Closing transient core [collection2]
   [junit4]   2> 1958194 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1958200 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.IndexSchema [collection9] Schema name=tiny
   [junit4]   2> 1958202 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1958204 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection9' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection9
   [junit4]   2> 1958204 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCore [[collection9] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection9], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection9/data/]
   [junit4]   2> 1958206 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=15, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1958253 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1958253 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1958254 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=42.0400390625, floorSegmentMB=0.6884765625, forceMergeDeletesPctAllowed=21.526417601931335, segmentsPerTier=49.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6268088795134374
   [junit4]   2> 1958255 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@6e26b4eb[collection9] main]
   [junit4]   2> 1958255 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection9/conf
   [junit4]   2> 1958256 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1958256 INFO  (searcherExecutor-6426-thread-1-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection9] Registered new searcher Searcher@6e26b4eb[collection9] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1958256 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCores Opening transient core collection9
   [junit4]   2> 1958256 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCores Closing transient core [collection3]
   [junit4]   2> 1958256 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCore [collection2]  CLOSING SolrCore org.apache.solr.core.SolrCore@750c11cc
   [junit4]   2> 1958258 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection2
   [junit4]   2> 1958258 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCore [collection3]  CLOSING SolrCore org.apache.solr.core.SolrCore@3fd1f6da
   [junit4]   2> 1958259 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection3
   [junit4]   2> 1958268 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1958274 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.IndexSchema [collection2] Schema name=tiny
   [junit4]   2> 1958276 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1958278 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection2
   [junit4]   2> 1958278 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection2/data/]
   [junit4]   2> 1958322 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1958322 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1958323 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=75.48828125, floorSegmentMB=1.6142578125, forceMergeDeletesPctAllowed=24.637811026252187, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1958324 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@1b360528[collection2] main]
   [junit4]   2> 1958324 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-002/collection2/conf
   [junit4]   2> 1958325 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1958325 INFO  (searcherExecutor-6430-thread-1-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@1b360528[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1958325 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCores Opening transient core collection2
   [junit4]   2> 1958325 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCores Closing transient core [collection6]
   [junit4]   2> 1958325 INFO  (Thread-8925) [    x:collection3] o.a.s.c.SolrCore [collection6]  CLOSING SolrCore org.apache.solr.core.SolrCore@59924a4e
   [junit4]   2> 1958325 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.S.Request [collection2]  webapp=null path=null params={q=*:*} hits=0 status=0 QTime=0
   [junit4]   2> 1958328 INFO  (Thread-8925) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection6
   [junit4]   2> 1958329 ERROR (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.SolrTestCaseJ4 REQUEST FAILED: xpath=//result[@numFound='10']
   [junit4]   2> 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]   2> <response>
   [junit4]   2> 
   [junit4]   2> <lst name="responseHeader">
   [junit4]   2>   <int name="status">0</int>
   [junit4]   2>   <int name="QTime">0</int>
   [junit4]   2>   <lst name="params">
   [junit4]   2>     <str name="q">*:*</str>
   [junit4]   2>   </lst>
   [junit4]   2> </lst>
   [junit4]   2> <result name="response" numFound="0" start="0">
   [junit4]   2> </result>
   [junit4]   2> </response>
   [junit4]   2> 
   [junit4]   2> 	request was:q=*:*
   [junit4]   2> 1958330 ERROR (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.SolrTestCaseJ4 REQUEST FAILED: q=*:*:java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound='10']
   [junit4]   2> 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]   2> <response>
   [junit4]   2> 
   [junit4]   2> <lst name="responseHeader">
   [junit4]   2>   <int name="status">0</int>
   [junit4]   2>   <int name="QTime">0</int>
   [junit4]   2>   <lst name="params">
   [junit4]   2>     <str name="q">*:*</str>
   [junit4]   2>   </lst>
   [junit4]   2> </lst>
   [junit4]   2> <result name="response" numFound="0" start="0">
   [junit4]   2> </result>
   [junit4]   2> </response>
   [junit4]   2> 
   [junit4]   2> 	request was:q=*:*
   [junit4]   2> 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:850)
   [junit4]   2> 	at org.apache.solr.core.TestLazyCores.check10(TestLazyCores.java:794)
   [junit4]   2> 	at org.apache.solr.core.TestLazyCores.testNoCommit(TestLazyCores.java:776)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 1958330 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.CoreContainer Shutting down CoreContainer instance=373826494
   [junit4]   2> 1958331 INFO  (coreCloseExecutor-6434-thread-1-processing-x:collection3) [    x:collection5] o.a.s.c.SolrCore [collection5]  CLOSING SolrCore org.apache.solr.core.SolrCore@3da67b00
   [junit4]   2> 1958331 INFO  (coreCloseExecutor-6434-thread-2-processing-x:collection3) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@55fa33c
   [junit4]   2> 1958332 INFO  (coreCloseExecutor-6434-thread-2-processing-x:collection3) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 1958332 INFO  (coreCloseExecutor-6434-thread-4-processing-x:collection3) [    x:collection8] o.a.s.c.SolrCore [collection8]  CLOSING SolrCore org.apache.solr.core.SolrCore@46f261d4
   [junit4]   2> 1958332 INFO  (coreCloseExecutor-6434-thread-3-processing-x:collection3) [    x:collection7] o.a.s.c.SolrCore [collection7]  CLOSING SolrCore org.apache.solr.core.SolrCore@7c19c62f
   [junit4]   2> 1958343 INFO  (coreCloseExecutor-6434-thread-5-processing-x:collection3) [    x:collection9] o.a.s.c.SolrCore [collection9]  CLOSING SolrCore org.apache.solr.core.SolrCore@7d4bac31
   [junit4]   2> 1958345 INFO  (coreCloseExecutor-6434-thread-3-processing-x:collection3) [    x:collection7] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection7
   [junit4]   2> 1958345 INFO  (coreCloseExecutor-6434-thread-1-processing-x:collection3) [    x:collection5] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection5
   [junit4]   2> 1958345 INFO  (coreCloseExecutor-6434-thread-5-processing-x:collection3) [    x:collection9] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection9
   [junit4]   2> 1958346 INFO  (coreCloseExecutor-6434-thread-4-processing-x:collection3) [    x:collection8] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection8
   [junit4]   2> 1958347 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1958347 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.SolrTestCaseJ4 ###Ending testNoCommit
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLazyCores -Dtests.method=testNoCommit -Dtests.seed=EBC9B0A6EFBEE1A3 -Dtests.slow=true -Dtests.locale=de-CH -Dtests.timezone=Atlantic/Bermuda -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.65s J0 | TestLazyCores.testNoCommit <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: Exception during query
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EBC9B0A6EFBEE1A3:34A9117724998206]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:857)
   [junit4]    > 	at org.apache.solr.core.TestLazyCores.check10(TestLazyCores.java:794)
   [junit4]    > 	at org.apache.solr.core.TestLazyCores.testNoCommit(TestLazyCores.java:776)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound='10']
   [junit4]    > 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]    > <response>
   [junit4]    > <lst name="responseHeader">
   [junit4]    >   <int name="status">0</int>
   [junit4]    >   <int name="QTime">0</int>
   [junit4]    >   <lst name="params">
   [junit4]    >     <str name="q">*:*</str>
   [junit4]    >   </lst>
   [junit4]    > </lst>
   [junit4]    > <result name="response" numFound="0" start="0">
   [junit4]    > </result>
   [junit4]    > </response>
   [junit4]    > 	request was:q=*:*
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:850)
   [junit4]    > 	... 41 more
   [junit4]   2> 1958354 INFO  (TEST-TestLazyCores.testRace-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.SolrTestCaseJ4 ###Starting testRace
   [junit4]   2> 1958375 INFO  (TEST-TestLazyCores.testRace-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.SolrCores Allocating transient cache for 4 transient cores
   [junit4]   2> 1958413 INFO  (coreLoadExecutor-6441-thread-2-processing-x:collection3) [    x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1958415 INFO  (coreLoadExecutor-6441-thread-1-processing-x:collection3) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1958415 INFO  (coreLoadExecutor-6441-thread-3-processing-x:collection3) [    x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1958421 INFO  (coreLoadExecutor-6441-thread-2-processing-x:collection3) [    x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
   [junit4]   2> 1958424 INFO  (coreLoadExecutor-6441-thread-2-processing-x:collection3) [    x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1958426 INFO  (coreLoadExecutor-6441-thread-1-processing-x:collection3) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
   [junit4]   2> 1958426 INFO  (coreLoadExecutor-6441-thread-3-processing-x:collection3) [    x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
   [junit4]   2> 1958426 INFO  (coreLoadExecutor-6441-thread-2-processing-x:collection3) [    x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-003/collection2
   [junit4]   2> 1958426 INFO  (coreLoadExecutor-6441-thread-2-processing-x:collection3) [    x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-003/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-003/collection2/data/]
   [junit4]   2> 1958437 INFO  (coreLoadExecutor-6441-thread-2-processing-x:collection3) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1958438 INFO  (coreLoadExecutor-6441-thread-3-processing-x:collection3) [    x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1958439 INFO  (coreLoadExecutor-6441-thread-1-processing-x:collection3) [    x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1958440 INFO  (coreLoadExecutor-6441-thread-3-processing-x:collection3) [    x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-003/collection5
   [junit4]   2> 1958440 INFO  (coreLoadExecutor-6441-thread-3-processing-x:collection3) [    x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-003/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-003/collection5/data/]
   [junit4]   2> 1958441 INFO  (coreLoadExecutor-6441-thread-1-processing-x:collection3) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-003/collection1
   [junit4]   2> 1958441 INFO  (coreLoadExecutor-6441-thread-1-processing-x:collection3) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-003/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-003/collection1/data/]
   [junit4]   2> 1958442 INFO  (coreLoadExecutor-6441-thread-3-processing-x:collection3) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1958443 INFO  (coreLoadExecutor-6441-thread-1-processing-x:collection3) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1958516 INFO  (coreLoadExecutor-6441-thread-1-processing-x:collection3) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1958516 INFO  (coreLoadExecutor-6441-thread-1-processing-x:collection3) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1958516 INFO  (coreLoadExecutor-6441-thread-1-processing-x:collection3) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=33.572265625, floorSegmentMB=0.2099609375, forceMergeDeletesPctAllowed=1.6390811190138121, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1958517 INFO  (coreLoadExecutor-6441-thread-1-processing-x:collection3) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@704204d1[collection1] main]
   [junit4]   2> 1958518 INFO  (coreLoadExecutor-6441-thread-1-processing-x:collection3) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-003/collection1/conf
   [junit4]   2> 1958518 INFO  (coreLoadExecutor-6441-thread-1-processing-x:collection3) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1958519 INFO  (searcherExecutor-6444-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@704204d1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1958545 INFO  (coreLoadExecutor-6441-thread-3-processing-x:collection3) [    x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1958545 INFO  (coreLoadExecutor-6441-thread-3-processing-x:collection3) [    x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1958546 INFO  (coreLoadExecutor-6441-thread-3-processing-x:collection3) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=33.572265625, floorSegmentMB=0.2099609375, forceMergeDeletesPctAllowed=1.6390811190138121, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1958546 INFO  (coreLoadExecutor-6441-thread-2-processing-x:collection3) [    x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1958546 INFO  (coreLoadExecutor-6441-thread-2-processing-x:collection3) [    x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1958547 INFO  (coreLoadExecutor-6441-thread-2-processing-x:collection3) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=33.572265625, floorSegmentMB=0.2099609375, forceMergeDeletesPctAllowed=1.6390811190138121, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1958547 INFO  (coreLoadExecutor-6441-thread-3-processing-x:collection3) [    x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@2bdfd830[collection5] main]
   [junit4]   2> 1958547 INFO  (coreLoadExecutor-6441-thread-3-processing-x:collection3) [    x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-003/collection5/conf
   [junit4]   2> 1958547 INFO  (coreLoadExecutor-6441-thread-3-processing-x:collection3) [    x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1958547 INFO  (coreLoadExecutor-6441-thread-2-processing-x:collection3) [    x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@566323f8[collection2] main]
   [junit4]   2> 1958548 INFO  (searcherExecutor-6443-thread-1-processing-x:collection5) [    x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@2bdfd830[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1958548 INFO  (coreLoadExecutor-6441-thread-2-processing-x:collection3) [    x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-003/collection2/conf
   [junit4]   2> 1958548 INFO  (coreLoadExecutor-6441-thread-2-processing-x:collection3) [    x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1958548 INFO  (searcherExecutor-6442-thread-1-processing-x:collection2) [    x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@566323f8[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1958548 INFO  (coreLoadExecutor-6441-thread-2-processing-x:collection3) [    x:collection2] o.a.s.c.SolrCores Opening transient core collection2
   [junit4]   2> 1958560 INFO  (Thread-8928) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1958567 INFO  (Thread-8928) [    x:collection3] o.a.s.s.IndexSchema [collection3] Schema name=tiny
   [junit4]   2> 1958569 INFO  (Thread-8928) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1958571 INFO  (Thread-8928) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection3' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-003/collection3
   [junit4]   2> 1958571 INFO  (Thread-8928) [    x:collection3] o.a.s.c.SolrCore [[collection3] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-003/collection3], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-003/collection3/data/]
   [junit4]   2> 1958573 INFO  (Thread-8928) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1958619 INFO  (Thread-8928) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1958619 INFO  (Thread-8928) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1958620 INFO  (Thread-8928) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=33.572265625, floorSegmentMB=0.2099609375, forceMergeDeletesPctAllowed=1.6390811190138121, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1958620 INFO  (Thread-8928) [    x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@71325233[collection3] main]
   [junit4]   2> 1958621 INFO  (Thread-8928) [    x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001/tempDir-003/collection3/conf
   [junit4]   2> 1958621 INFO  (Thread-8928) [    x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1958621 INFO  (searcherExecutor-6454-thread-1-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection3] Registered new searcher Searcher@71325233[collection3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1958621 INFO  (Thread-8928) [    x:collection3] o.a.s.c.SolrCores Opening transient core collection3
   [junit4]   2> 1958629 INFO  (TEST-TestLazyCores.testRace-seed#[EBC9B0A6EFBEE1A3]) [    x:collection3] o.a.s.c.CoreContainer Shutting down CoreContainer instance=2001606693
   [junit4]   2> 1958630 INFO  (coreCloseExecutor-6458-thread-1-processing-x:collection3) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@6adacd69
   [junit4]   2> 1958630 INFO  (coreCloseExecutor-6458-thread-2-processing-x:collection3) [    x:collection5] o.a.s.c.SolrCore [collection5]  CLOSING SolrCore org.apache.solr.core.SolrCore@13b91bf4
   [junit4]   2> 1958631 INFO  (coreCloseExecutor-6458-thread-3-processing-x:collection3) [    x:collection2] o.a.s.c.SolrCore [collection2]  CLOSING SolrCore org.apache.solr.core.SolrCore@15cab519
   [junit4]   2> 1958631 INFO  (coreCloseExecutor-6458-thread-1-processing-x:collection3) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 1958631 INFO  (coreCloseExecutor-6458-thread-4-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection3]  CLOSING SolrCore org.apache.solr.core.SolrCore@513be07
   [junit4]   2> 1958632 INFO  (coreCloseExecutor-6458-thread-3-processing-x:collection3) [    x:collection2] o.a.s.m.SolrMetricManager Closing metric repor

[...truncated too long message...]

porary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBC9B0A6EFBEE1A3-001
   [junit4]   2> Feb 02, 2017 5:03:02 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> Feb 02, 2017 5:03:22 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   [junit4]   2>    1) Thread[id=21305, name=searcherExecutor-6430-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Feb 02, 2017 5:03:22 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=21305, name=searcherExecutor-6430-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2> Feb 02, 2017 5:03:25 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=21305, name=searcherExecutor-6430-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {v_t=Lucene50(blocksize=128), id=Lucene50(blocksize=128), v_s=BlockTreeOrds(blocksize=128), v_f=PostingsFormat(name=Memory)}, docValues:{}, maxPointsInLeafNode=1076, maxMBSortInHeap=6.2177947092573795, sim=RandomSimilarity(queryNorm=false): {}, locale=de-CH, timezone=Atlantic/Bermuda
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=2,free=279777584,total=532676608
   [junit4]   2> NOTE: All tests run in this JVM: [TestOnReconnectListenerSupport, DirectSolrSpellCheckerTest, ClusterStateUpdateTest, TestPartialUpdateDeduplication, OverseerTest, SynonymTokenizerTest, TestCloudDeleteByQuery, AnalyticsMergeStrategyTest, TestEmbeddedSolrServerConstructors, MetricsHandlerTest, HdfsUnloadDistributedZkTest, TestWordDelimiterFilterFactory, SolrXmlInZkTest, SolrMetricsIntegrationTest, OverseerRolesTest, CdcrVersionReplicationTest, CoreMergeIndexesAdminHandlerTest, DistribCursorPagingTest, UUIDUpdateProcessorFallbackTest, HdfsCollectionsAPIDistributedZkTest, TestMacros, TestLRUCache, TestRTGBase, TestBM25SimilarityFactory, TestQuerySenderNoQuery, CollectionReloadTest, SharedFSAutoReplicaFailoverUtilsTest, TestRandomDVFaceting, TestSizeLimitedDistributedMap, SegmentsInfoRequestHandlerTest, TestSchemaNameResource, ManagedSchemaRoundRobinCloudTest, TestIBSimilarityFactory, SearchHandlerTest, DistributedFacetPivotSmallTest, TestFieldCacheSanityChecker, SolrTestCaseJ4Test, TestSolrQueryResponse, CoreSorterTest, TestHashPartitioner, TestApiFramework, SolrCoreTest, SSLMigrationTest, DistributedFacetExistsSmallTest, DocumentBuilderTest, TestFieldSortValues, TestExceedMaxTermLength, TestManagedStopFilterFactory, SpatialFilterTest, TestExtendedDismaxParser, DocValuesTest, TestConfigSetProperties, FullHLLTest, TestSolrCloudWithKerberosAlt, TestSubQueryTransformerDistrib, TestCloudSchemaless, BadCopyFieldTest, TestNonDefinedSimilarityFactory, HdfsLockFactoryTest, HttpSolrCallGetCoreTest, TestConfigSetsAPIExclusivity, DistributedFacetPivotLongTailTest, ClassificationUpdateProcessorFactoryTest, TestCopyFieldCollectionResource, PrimitiveFieldTypeTest, TestJmxIntegration, MBeansHandlerTest, TestStressReorder, TestRequestStatusCollectionAPI, SyncSliceTest, TestStressLiveNodes, DistributedVersionInfoTest, CollectionTooManyReplicasTest, SolrCLIZkUtilsTest, XsltUpdateRequestHandlerTest, TestIntervalFaceting, TestReplicationHandlerBackup, DataDrivenBlockJoinTest, SystemInfoHandlerTest, CloudMLTQParserTest, CursorPagingTest, HdfsRestartWhileUpdatingTest, SolrIndexConfigTest, TestFastLRUCache, DistributedSpellCheckComponentTest, OutputWriterTest, TestInfoStreamLogging, FastVectorHighlighterTest, RecoveryZkTest, TestSegmentSorting, TestLockTree, AnalysisErrorHandlingTest, TestCoreContainer, TestCollapseQParserPlugin, PrimUtilsTest, TestNRTOpen, TestSolrDeletionPolicy1, SchemaVersionSpecificBehaviorTest, TestManagedSchema, PolyFieldTest, CollectionsAPISolrJTest, TestStandardQParsers, TestFieldCache, CdcrBootstrapTest, TriLevelCompositeIdRoutingTest, TestNamedUpdateProcessors, TestSchemaManager, BlockCacheTest, TestRecoveryHdfs, DirectSolrConnectionTest, TestCloudRecovery, TestJsonFacets, TestManagedSchemaThreadSafety, MultiTermTest, TestSchemaSimilarityResource, ShardSplitTest, TestLeaderInitiatedRecoveryThread, LeaderInitiatedRecoveryOnShardRestartTest, TermsComponentTest, StatelessScriptUpdateProcessorFactoryTest, CheckHdfsIndexTest, OverseerTaskQueueTest, TestCorePropertiesReload, MetricUtilsTest, TestSubQueryTransformer, TestUninvertingReader, DistributedDebugComponentTest, TestFaceting, TestNoOpRegenerator, TestTolerantSearch, TestLeaderElectionWithEmptyReplica, DateFieldTest, TestCustomDocTransformer, TestSha256AuthenticationProvider, TestCharFilters, DocumentAnalysisRequestHandlerTest, TestCloudPivotFacet, CurrencyFieldXmlFileTest, CopyFieldTest, TestPerFieldSimilarity, SpatialHeatmapFacetsTest, TestTestInjection, HdfsThreadLeakTest, TestCSVResponseWriter, SuggestComponentTest, RequiredFieldsTest, JvmMetricsTest, ParsingFieldUpdateProcessorsTest, JsonValidatorTest, DistributedMLTComponentTest, TestCursorMarkWithoutUniqueKey, TestXmlQParserPlugin, CursorMarkTest, DisMaxRequestHandlerTest, CleanupOldIndexTest, StressHdfsTest, HdfsRecoverLeaseTest, BigEndianAscendingWordDeserializerTest, LeaderElectionContextKeyTest, ReplicaListTransformerTest, ConnectionReuseTest, SimplePostToolTest, FileUtilsTest, TestLazyCores]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLazyCores -Dtests.seed=EBC9B0A6EFBEE1A3 -Dtests.slow=true -Dtests.locale=de-CH -Dtests.timezone=Atlantic/Bermuda -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | TestLazyCores (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   [junit4]    >    1) Thread[id=21305, name=searcherExecutor-6430-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EBC9B0A6EFBEE1A3]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=21305, name=searcherExecutor-6430-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EBC9B0A6EFBEE1A3]:0)
   [junit4] Completed [347/691 (1!)] on J0 in 30.82s, 9 tests, 3 errors <<< FAILURES!

[...truncated 63526 lines...]


[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1111 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1111/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC

1 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:     1) Thread[id=8143, name=jetty-launcher-1642-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)    2) Thread[id=8145, name=jetty-launcher-1642-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   1) Thread[id=8143, name=jetty-launcher-1642-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   2) Thread[id=8145, name=jetty-launcher-1642-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
	at __randomizedtesting.SeedInfo.seed([4017E5C79AB987F1]:0)




Build Log:
[...truncated 11283 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_4017E5C79AB987F1-001/init-core-data-001
   [junit4]   2> 684907 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[4017E5C79AB987F1]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields
   [junit4]   2> 684909 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[4017E5C79AB987F1]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 684939 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[4017E5C79AB987F1]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_4017E5C79AB987F1-001/tempDir-001
   [junit4]   2> 684939 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[4017E5C79AB987F1]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 684939 INFO  (Thread-1931) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 684939 INFO  (Thread-1931) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 685040 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[4017E5C79AB987F1]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:44557
   [junit4]   2> 685048 INFO  (jetty-launcher-1642-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 685048 INFO  (jetty-launcher-1642-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 685050 INFO  (jetty-launcher-1642-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4355c1d7{/solr,null,AVAILABLE}
   [junit4]   2> 685050 INFO  (jetty-launcher-1642-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2e3e69ec{/solr,null,AVAILABLE}
   [junit4]   2> 685051 INFO  (jetty-launcher-1642-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@a809681{HTTP/1.1,[http/1.1]}{127.0.0.1:52338}
   [junit4]   2> 685051 INFO  (jetty-launcher-1642-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3164d784{HTTP/1.1,[http/1.1]}{127.0.0.1:49782}
   [junit4]   2> 685051 INFO  (jetty-launcher-1642-thread-1) [    ] o.e.j.s.Server Started @688867ms
   [junit4]   2> 685051 INFO  (jetty-launcher-1642-thread-2) [    ] o.e.j.s.Server Started @688867ms
   [junit4]   2> 685051 INFO  (jetty-launcher-1642-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=52338}
   [junit4]   2> 685051 INFO  (jetty-launcher-1642-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=49782}
   [junit4]   2> 685051 ERROR (jetty-launcher-1642-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 685051 ERROR (jetty-launcher-1642-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 685051 INFO  (jetty-launcher-1642-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 685051 INFO  (jetty-launcher-1642-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 685051 INFO  (jetty-launcher-1642-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 685051 INFO  (jetty-launcher-1642-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 685051 INFO  (jetty-launcher-1642-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 685051 INFO  (jetty-launcher-1642-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 685051 INFO  (jetty-launcher-1642-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-01T18:25:50.171Z
   [junit4]   2> 685051 INFO  (jetty-launcher-1642-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-01T18:25:50.171Z
   [junit4]   2> 685056 INFO  (jetty-launcher-1642-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 685056 INFO  (jetty-launcher-1642-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 685062 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159faecf3ed0002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 685062 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159faecf3ed0001, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 685064 INFO  (jetty-launcher-1642-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44557/solr
   [junit4]   2> 685064 INFO  (jetty-launcher-1642-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44557/solr
   [junit4]   2> 685106 INFO  (jetty-launcher-1642-thread-1) [n:127.0.0.1:52338_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:52338_solr
   [junit4]   2> 685107 INFO  (jetty-launcher-1642-thread-2) [n:127.0.0.1:49782_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49782_solr
   [junit4]   2> 685108 INFO  (jetty-launcher-1642-thread-1) [n:127.0.0.1:52338_solr    ] o.a.s.c.Overseer Overseer (id=97384762576797701-127.0.0.1:52338_solr-n_0000000000) starting
   [junit4]   2> 685111 INFO  (zkCallback-1653-thread-1-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 685112 INFO  (zkCallback-1652-thread-1-processing-n:127.0.0.1:52338_solr) [n:127.0.0.1:52338_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 685138 INFO  (jetty-launcher-1642-thread-1) [n:127.0.0.1:52338_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52338_solr
   [junit4]   2> 685151 INFO  (zkCallback-1652-thread-1-processing-n:127.0.0.1:52338_solr) [n:127.0.0.1:52338_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 685151 INFO  (zkCallback-1653-thread-1-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 685181 INFO  (jetty-launcher-1642-thread-2) [n:127.0.0.1:49782_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 685194 INFO  (jetty-launcher-1642-thread-2) [n:127.0.0.1:49782_solr    ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=solaris-vm,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
   [junit4]   2> 685230 INFO  (jetty-launcher-1642-thread-1) [n:127.0.0.1:52338_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 685231 INFO  (jetty-launcher-1642-thread-1) [n:127.0.0.1:52338_solr    ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=solaris-vm,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
   [junit4]   2> 685287 INFO  (jetty-launcher-1642-thread-1) [n:127.0.0.1:52338_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 685290 INFO  (jetty-launcher-1642-thread-2) [n:127.0.0.1:49782_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 685315 INFO  (jetty-launcher-1642-thread-1-EventThread) [n:127.0.0.1:52338_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 685315 INFO  (jetty-launcher-1642-thread-2-EventThread) [n:127.0.0.1:49782_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 685495 INFO  (jetty-launcher-1642-thread-2) [n:127.0.0.1:49782_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_4017E5C79AB987F1-001/tempDir-001/node2/.
   [junit4]   2> 685495 INFO  (jetty-launcher-1642-thread-1) [n:127.0.0.1:52338_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_4017E5C79AB987F1-001/tempDir-001/node1/.
   [junit4]   2> 686008 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidGroup
   [junit4]   2> 686043 WARN  (qtp1745506118-8103) [    ] o.a.h.s.ShellBasedUnixGroupsMapping got exception trying to get groups for user bar: id: invalid user name: "bar"
   [junit4]   2> 
   [junit4]   2> 686043 WARN  (qtp1745506118-8103) [    ] o.a.h.s.UserGroupInformation No groups available for user bar
   [junit4]   2> 686078 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[4017E5C79AB987F1]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bogusGroup&p=bogusGroup&t=test&e=1486009551143&s=gJpCGkC4hE6eIadBRAo0QcExYoc="; Path=/; Domain=127.0.0.1; Expires= et, 02-feb.-2017 04:25:51 GMT; HttpOnly". Invalid 'expires' attribute: et, 02-feb.-2017 04:25:51 GMT
   [junit4]   2> 686078 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidGroup
   [junit4]   2> 686100 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testForwarding
   [junit4]   2> 686730 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[4017E5C79AB987F1]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 686732 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[4017E5C79AB987F1]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44557/solr ready
   [junit4]   2> 686741 INFO  (qtp1187429479-8099) [n:127.0.0.1:52338_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 686744 INFO  (OverseerThreadFactory-3506-thread-1-processing-n:127.0.0.1:52338_solr) [n:127.0.0.1:52338_solr    ] o.a.s.c.CreateCollectionCmd Create collection forwardingCollection
   [junit4]   2> 686854 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2
   [junit4]   2> 686961 INFO  (zkCallback-1653-thread-1-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 687875 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 687916 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.IndexSchema [forwardingCollection_shard1_replica1] Schema name=test
   [junit4]   2> 688007 WARN  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.IndexSchema [forwardingCollection_shard1_replica1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 688009 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 688027 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'forwardingCollection_shard1_replica1' using configuration from collection forwardingCollection
   [junit4]   2> 688027 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SolrCore [[forwardingCollection_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_4017E5C79AB987F1-001/tempDir-001/node2/forwardingCollection_shard1_replica1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_4017E5C79AB987F1-001/tempDir-001/node2/./forwardingCollection_shard1_replica1/data/]
   [junit4]   2> 688027 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c53752e
   [junit4]   2> 688029 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1511884750, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 688082 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 688082 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 688083 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 688083 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 688084 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5965175442032566]
   [junit4]   2> 688084 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@78b5abf4[forwardingCollection_shard1_replica1] main]
   [junit4]   2> 688086 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 688092 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 688109 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 688109 INFO  (searcherExecutor-3511-thread-1-processing-n:127.0.0.1:49782_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 688109 INFO  (searcherExecutor-3511-thread-1-processing-n:127.0.0.1:49782_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 688109 INFO  (searcherExecutor-3511-thread-1-processing-n:127.0.0.1:49782_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 688109 INFO  (searcherExecutor-3511-thread-1-processing-n:127.0.0.1:49782_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 688110 INFO  (searcherExecutor-3511-thread-1-processing-n:127.0.0.1:49782_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 688110 INFO  (searcherExecutor-3511-thread-1-processing-n:127.0.0.1:49782_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 688110 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1558156204551700480
   [junit4]   2> 688110 INFO  (searcherExecutor-3511-thread-1-processing-n:127.0.0.1:49782_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 688111 INFO  (searcherExecutor-3511-thread-1-processing-n:127.0.0.1:49782_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 688111 INFO  (searcherExecutor-3511-thread-1-processing-n:127.0.0.1:49782_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 688112 INFO  (searcherExecutor-3511-thread-1-processing-n:127.0.0.1:49782_solr x:forwardingCollection_shard1_replica1 s:shard1 c:forwardingCollection) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica1] Registered new searcher Searcher@78b5abf4[forwardingCollection_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 688121 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 688121 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 688121 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:49782/solr/forwardingCollection_shard1_replica1/
   [junit4]   2> 688121 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 688121 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:49782/solr/forwardingCollection_shard1_replica1/ has no replicas
   [junit4]   2> 688128 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:49782/solr/forwardingCollection_shard1_replica1/ shard1
   [junit4]   2> 688233 INFO  (zkCallback-1653-thread-1-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 688281 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 688284 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1  x:forwardingCollection_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2} status=0 QTime=1431
   [junit4]   2> 688286 WARN  (httpShardExecutor-3500-thread-1-processing-n:127.0.0.1:52338_solr [http:////127.0.0.1:49782//solr] http:////127.0.0.1:49782//solr) [n:127.0.0.1:52338_solr    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=$&p=$&t=test&e=1486009551973&s=veoSPaI2AHDZRiXyxEF+4xd7ebY="; Path=/; Domain=127.0.0.1; Expires= et, 02-feb.-2017 04:25:51 GMT; HttpOnly". Invalid 'expires' attribute: et, 02-feb.-2017 04:25:51 GMT
   [junit4]   2> 688292 INFO  (qtp1187429479-8099) [n:127.0.0.1:52338_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 688387 INFO  (zkCallback-1653-thread-1-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 688747 INFO  (OverseerCollectionConfigSetProcessor-97384762576797701-127.0.0.1:52338_solr-n_0000000000) [n:127.0.0.1:52338_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 689294 INFO  (qtp1187429479-8099) [n:127.0.0.1:52338_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2555
   [junit4]   2> 689295 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[4017E5C79AB987F1]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=user&p=user&t=test&e=1486009551856&s=GbcEK8wi6x3BY2rvFaPUy9h7M8s="; Path=/; Domain=127.0.0.1; Expires= et, 02-feb.-2017 04:25:51 GMT; HttpOnly". Invalid 'expires' attribute: et, 02-feb.-2017 04:25:51 GMT
   [junit4]   2> 689295 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[4017E5C79AB987F1]) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: forwardingCollection failOnTimeout:true timeout (sec):100
   [junit4]   2> 689296 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[4017E5C79AB987F1]) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: forwardingCollection
   [junit4]   2> 689301 INFO  (qtp1745506118-8110) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.c.S.Request [forwardingCollection_shard1_replica1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 689301 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[4017E5C79AB987F1]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=user&p=user&t=test&e=1486009554420&s=Cv9x/25W4lAUxqW64Nvvy+KpuwA="; Path=/; Domain=127.0.0.1; Expires= et, 02-feb.-2017 04:25:54 GMT; HttpOnly". Invalid 'expires' attribute: et, 02-feb.-2017 04:25:54 GMT
   [junit4]   2> 689308 INFO  (qtp1745506118-8103) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.c.S.Request [forwardingCollection_shard1_replica1]  webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 689308 WARN  (qtp1187429479-8106) [n:127.0.0.1:52338_solr    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=$&p=$&t=test&e=1486009554427&s=VhALQMe5ogOcQoj4h7gxQr3lbrA="; Path=/; Domain=127.0.0.1; Expires= et, 02-feb.-2017 04:25:54 GMT; HttpOnly". Invalid 'expires' attribute: et, 02-feb.-2017 04:25:54 GMT
   [junit4]   2> 689309 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[4017E5C79AB987F1]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=user&p=user&t=test&e=1486009554424&s=Kqo7o+UAP7UEIxY8qN2Y1lISjHQ="; Path=/; Domain=127.0.0.1; Expires= et, 02-feb.-2017 04:25:54 GMT; HttpOnly". Invalid 'expires' attribute: et, 02-feb.-2017 04:25:54 GMT
   [junit4]   2> 689309 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[4017E5C79AB987F1]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=$&p=$&t=test&e=1486009554427&s=VhALQMe5ogOcQoj4h7gxQr3lbrA="; Path=/; Domain=127.0.0.1; Expires= et, 02-feb.-2017 04:25:54 GMT; HttpOnly". Invalid 'expires' attribute: et, 02-feb.-2017 04:25:54 GMT
   [junit4]   2> 689309 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testForwarding
   [junit4]   2> 689341 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigHosts
   [junit4]   2> 689350 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[4017E5C79AB987F1]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noHosts&p=noHosts&t=test&e=1486009554463&s=I+y6bhwaHu7kbp7257ifSt3EYy0="; Path=/; Domain=127.0.0.1; Expires= et, 02-feb.-2017 04:25:54 GMT; HttpOnly". Invalid 'expires' attribute: et, 02-feb.-2017 04:25:54 GMT
   [junit4]   2> 689350 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigHosts
   [junit4]   2> 689384 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateAnyHostAnyUser
   [junit4]   2> 689386 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 689386 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 689386 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[4017E5C79AB987F1]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostAnyUser&p=anyHostAnyUser&t=test&e=1486009554505&s=tfQv6jaMv6CksR0yyJtS0xPjz8Y="; Path=/; Domain=127.0.0.1; Expires= et, 02-feb.-2017 04:25:54 GMT; HttpOnly". Invalid 'expires' attribute: et, 02-feb.-2017 04:25:54 GMT
   [junit4]   2> 689387 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateAnyHostAnyUser
   [junit4]   2> 689409 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyWrongHost
   [junit4]   2> 709437 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159faecf3ed0005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 709438 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[4017E5C79AB987F1]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=wrongHost&p=wrongHost&t=test&e=1486009554529&s=LJcVMB6eTIafJ42z4EXZT8dM2qU="; Path=/; Domain=127.0.0.1; Expires= et, 02-feb.-2017 04:25:54 GMT; HttpOnly". Invalid 'expires' attribute: et, 02-feb.-2017 04:25:54 GMT
   [junit4]   2> 709439 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyWrongHost
   [junit4]   2> 709473 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigGroups
   [junit4]   2> 709476 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[4017E5C79AB987F1]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noGroups&p=noGroups&t=test&e=1486009574594&s=a3p4VxxQiTa2jAtwfXiZ5//2jF0="; Path=/; Domain=127.0.0.1; Expires= et, 02-feb.-2017 04:26:14 GMT; HttpOnly". Invalid 'expires' attribute: et, 02-feb.-2017 04:26:14 GMT
   [junit4]   2> 709476 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigGroups
   [junit4]   2> 709513 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidRemote
   [junit4]   2> 709520 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[4017E5C79AB987F1]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=test&e=1486009574634&s=tQ87aIgqcrVsE4UWYgySzvOKnow="; Path=/; Domain=127.0.0.1; Expires= et, 02-feb.-2017 04:26:14 GMT; HttpOnly". Invalid 'expires' attribute: et, 02-feb.-2017 04:26:14 GMT
   [junit4]   2> 709520 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidRemote
   [junit4]   2> 709540 WARN  (zkCallback-1660-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3d0fdd06 name: ZooKeeperConnection Watcher:127.0.0.1:44557/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 709540 WARN  (zkCallback-1660-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 709540 WARN  (zkCallback-1652-thread-4-processing-n:127.0.0.1:52338_solr) [n:127.0.0.1:52338_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@79c45c02 name: ZooKeeperConnection Watcher:127.0.0.1:44557/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 709540 WARN  (zkCallback-1652-thread-4-processing-n:127.0.0.1:52338_solr) [n:127.0.0.1:52338_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 709541 WARN  (zkCallback-1653-thread-3-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2186d55f name: ZooKeeperConnection Watcher:127.0.0.1:44557/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 709541 WARN  (zkCallback-1653-thread-3-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 709558 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidProxyUser
   [junit4]   2> 709561 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[4017E5C79AB987F1]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bar&p=bar&t=test&e=1486009574678&s=jf6UuzOzy6hVNOTSNio6Iu8p6/I="; Path=/; Domain=127.0.0.1; Expires= et, 02-feb.-2017 04:26:14 GMT; HttpOnly". Invalid 'expires' attribute: et, 02-feb.-2017 04:26:14 GMT
   [junit4]   2> 709561 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidProxyUser
   [junit4]   2> 709598 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyUnknownRemote
   [junit4]   2> 709642 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[4017E5C79AB987F1]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=test&e=1486009574718&s=FC8WuDPiwh4UTb2JA1+d5hBDU1w="; Path=/; Domain=127.0.0.1; Expires= et, 02-feb.-2017 04:26:14 GMT; HttpOnly". Invalid 'expires' attribute: et, 02-feb.-2017 04:26:14 GMT
   [junit4]   2> 709643 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyUnknownRemote
   [junit4]   2> 709679 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateGroup
   [junit4]   2> 709710 INFO  (qtp1745506118-8104) [n:127.0.0.1:49782_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=jenkins&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 709710 INFO  (qtp1745506118-8104) [n:127.0.0.1:49782_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=jenkins&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 709711 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[4017E5C79AB987F1]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostUsersGroup&p=anyHostUsersGroup&t=test&e=1486009574800&s=eILymkd4LskCnynTOIJ0Tm8ad6M="; Path=/; Domain=127.0.0.1; Expires= et, 02-feb.-2017 04:26:14 GMT; HttpOnly". Invalid 'expires' attribute: et, 02-feb.-2017 04:26:14 GMT
   [junit4]   2> 709711 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateGroup
   [junit4]   2> 709747 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNullProxyUser
   [junit4]   2> 709748 INFO  (qtp1745506118-8105) [n:127.0.0.1:49782_solr    ] o.a.s.s.SolrDispatchFilter Error authenticating
   [junit4]   2> java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:213)
   [junit4]   2> 	at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> 	at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:519)
   [junit4]   2> 	at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:128)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
   [junit4]   2> 	at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:243)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:416)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:312)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:298)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 709749 WARN  (qtp1745506118-8105) [n:127.0.0.1:49782_solr    ] o.e.j.s.ServletHandler /solr/admin/collections
   [junit4]   2> org.apache.solr.common.SolrException: Error during request authentication, 
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:422)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:312)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:298)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:213)
   [junit4]   2> 	at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> 	at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:519)
   [junit4]   2> 	at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:128)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
   [junit4]   2> 	at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:243)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:416)
   [junit4]   2> 	... 26 more
   [junit4]   2> 709750 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNullProxyUser
   [junit4]   2> 709788 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateHost
   [junit4]   2> 709789 INFO  (qtp1745506118-8100) [n:127.0.0.1:49782_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 709789 INFO  (qtp1745506118-8100) [n:127.0.0.1:49782_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 709790 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[4017E5C79AB987F1]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=test&e=1486009574909&s=sNTfKqkX8HnC+/US6UGtPhaaytA="; Path=/; Domain=127.0.0.1; Expires= et, 02-feb.-2017 04:26:14 GMT; HttpOnly". Invalid 'expires' attribute: et, 02-feb.-2017 04:26:14 GMT
   [junit4]   2> 709790 INFO  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[4017E5C79AB987F1]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateHost
   [junit4]   2> 710563 WARN  (zkCallback-1660-thread-3) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3d0fdd06 name: ZooKeeperConnection Watcher:127.0.0.1:44557/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
   [junit4]   2> 710563 WARN  (zkCallback-1660-thread-3) [    ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
   [junit4]   2> 710563 WARN  (zkCallback-1660-thread-3) [    ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
   [junit4]   2> 710564 INFO  (zkCallback-1660-thread-3) [    ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
   [junit4]   2> 710664 ERROR (zkCallback-1660-thread-3) [    ] o.a.s.c.c.ZkStateReader A ZK error has occurred
   [junit4]   2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:322)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:319)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:319)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:374)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader$1.command(ZkStateReader.java:232)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168)
   [junit4]   2> 	at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:268)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 710664 ERROR (zkCallback-1660-thread-3) [    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException: A ZK error has occurred
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader$1.command(ZkStateReader.java:235)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:168)
   [junit4]   2> 	at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
   [junit4]   2> 	at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:142)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:268)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:322)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:319)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:319)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:374)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader$1.command(ZkStateReader.java:232)
   [junit4]   2> 	... 10 more
   [junit4]   2> 
   [junit4]   2> 710664 WARN  (zkCallback-1660-thread-3) [    ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed
   [junit4]   2> 710664 INFO  (zkCallback-1660-thread-3) [    ] o.a.s.c.c.ConnectionManager Connected:true
   [junit4]   2> 710666 INFO  (jetty-launcher-1642-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@a809681{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 710666 INFO  (jetty-launcher-1642-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1033623026
   [junit4]   2> 710667 INFO  (jetty-launcher-1642-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@3164d784{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 710667 INFO  (jetty-launcher-1642-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1932775437
   [junit4]   2> 710854 WARN  (zkCallback-1653-thread-4-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2186d55f name: ZooKeeperConnection Watcher:127.0.0.1:44557/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
   [junit4]   2> 710854 WARN  (zkCallback-1653-thread-4-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
   [junit4]   2> 710855 WARN  (zkCallback-1653-thread-4-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
   [junit4]   2> 710856 INFO  (zkCallback-1653-thread-4-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
   [junit4]   2> 710856 INFO  (zkCallback-1653-thread-4-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
   [junit4]   2> 710858 INFO  (zkCallback-1653-thread-4-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 710868 INFO  (zkCallback-1653-thread-4-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49782_solr
   [junit4]   2> 710869 INFO  (zkCallback-1653-thread-2-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 710872 INFO  (coreZkRegister-3501-thread-1-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.ZkController Registering core forwardingCollection_shard1_replica1 afterExpiration? true
   [junit4]   2> 710873 INFO  (zkCallback-1653-thread-4-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.c.DefaultConnectionStrategy Reconnected to ZooKeeper
   [junit4]   2> 710873 INFO  (zkCallback-1653-thread-4-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.c.ConnectionManager Connected:true
   [junit4]   2> 710874 INFO  (Thread-1958) [n:127.0.0.1:49782_solr    ] o.a.s.c.SolrCore config update listener called for core forwardingCollection_shard1_replica1
   [junit4]   2> 710874 INFO  (coreZkRegister-3501-thread-1-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.c.ActionThrottle The last leader attempt started 22768ms ago.
   [junit4]   2> 710877 INFO  (coreZkRegister-3501-thread-1-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 710877 INFO  (coreZkRegister-3501-thread-1-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:49782/solr/forwardingCollection_shard1_replica1/
   [junit4]   2> 710877 INFO  (coreZkRegister-3501-thread-1-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 710877 INFO  (coreZkRegister-3501-thread-1-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:49782/solr/forwardingCollection_shard1_replica1/ has no replicas
   [junit4]   2> 710889 INFO  (coreZkRegister-3501-thread-1-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:49782/solr/forwardingCollection_shard1_replica1/ shard1
   [junit4]   2> 710890 INFO  (coreZkRegister-3501-thread-1-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 710954 ERROR (jetty-launcher-1642-thread-1) [    ] o.a.s.c.Overseer Could not create Overseer node
   [junit4]   2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$9.execute(SolrZkClient.java:391)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$9.execute(SolrZkClient.java:388)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:388)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer.createOverseerNode(Overseer.java:728)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer.getStateUpdateQueue(Overseer.java:601)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer.getStateUpdateQueue(Overseer.java:588)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.publishNodeAsDown(ZkController.java:2407)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.shutdown(CoreContainer.java:666)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.destroy(SolrDispatchFilter.java:289)
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.destroyInstance(FilterHolder.java:172)
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doStop(FilterHolder.java:150)
   [junit4]   2> 	at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:89)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doStop(ServletHandler.java:238)
   [junit4]   2> 	at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:89)
   [junit4]   2> 	at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:142)
   [junit4]   2> 	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:160)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.AbstractHandler.doStop(AbstractHandler.java:73)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doStop(SessionHandler.java:127)
   [junit4]   2> 	at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:89)
   [junit4]   2> 	at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:142)
   [junit4]   2> 	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:160)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.AbstractHandler.doStop(AbstractHandler.java:73)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.stopContext(ContextHandler.java:825)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletContextHandler.stopContext(ServletContextHandler.java:356)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doStop(ContextHandler.java:880)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletContextHandler.doStop(ServletContextHandler.java:272)
   [junit4]   2> 	at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:89)
   [junit4]   2> 	at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:142)
   [junit4]   2> 	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:160)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.AbstractHandler.doStop(AbstractHandler.java:73)
   [junit4]   2> 	at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:89)
   [junit4]   2> 	at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:142)
   [junit4]   2> 	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:160)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.AbstractHandler.doStop(AbstractHandler.java:73)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.doStop(Server.java:493)
   [junit4]   2> 	at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:89)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner.stop(JettySolrRunner.java:428)
   [junit4]   2> 	at org.apache.solr.cloud.MiniSolrCloudCluster.stopJettySolrRunner(MiniSolrCloudCluster.java:415)
   [junit4]   2> 	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$shutdown$1(MiniSolrCloudCluster.java:450)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 710954 WARN  (jetty-launcher-1642-thread-1) [    ] o.a.s.c.ZkController Could not publish node as down: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer
   [junit4]   2> 710955 INFO  (coreCloseExecutor-3516-thread-1) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@7ff64674
   [junit4]   2> 710989 INFO  (coreCloseExecutor-3516-thread-1) [n:127.0.0.1:49782_solr c:forwardingCollection s:shard1 r:core_node1 x:forwardingCollection_shard1_replica1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.forwardingCollection.shard1.replica1
   [junit4]   2> 711013 INFO  (zkCallback-1652-thread-1-processing-n:127.0.0.1:52338_solr) [n:127.0.0.1:52338_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 711016 INFO  (jetty-launcher-1642-thread-2) [    ] o.a.s.c.Overseer Overseer (id=97384762576797701-127.0.0.1:52338_solr-n_0000000000) closing
   [junit4]   2> 711016 INFO  (OverseerStateUpdate-97384762576797701-127.0.0.1:52338_solr-n_0000000000) [n:127.0.0.1:52338_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:52338_solr
   [junit4]   2> 711019 ERROR (OverseerHdfsCoreFailoverThread-97384762576797701-127.0.0.1:52338_solr-n_0000000000) [n:127.0.0.1:52338_solr    ] o.a.s.c.OverseerAutoReplicaFailoverThread OverseerAutoReplicaFailoverThread had an error in its thread work loop.:org.apache.solr.common.SolrException: Could not load collection from ZK: forwardingCollection
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1101)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:638)
   [junit4]   2> 	at org.apache.solr.common.cloud.ClusterState.getCollectionsMap(ClusterState.java:250)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.doWork(OverseerAutoReplicaFailoverThread.java:166)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:131)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 	at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:353)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1110)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1096)
   [junit4]   2> 	... 5 more
   [junit4]   2> 
   [junit4]   2> 711021 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159faecf3ed0005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 711024 INFO  (jetty-launcher-1642-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 711025 ERROR (Thread[Thread-1948,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:52338_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 711561 INFO  (jetty-launcher-1642-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4355c1d7{/solr,null,UNAVAILABLE}
   [junit4]   2> 712493 WARN  (zkCallback-1653-thread-4-processing-n:127.0.0.1:49782_solr) [n:127.0.0.1:49782_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 712494 INFO  (jetty-launcher-1642-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 712494 ERROR (Thread[Thread-1950,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:49782_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 712501 INFO  (jetty-launcher-1642-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2e3e69ec{/solr,null,UNAVAILABLE}
   [junit4]   2> 712503 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[4017E5C79AB987F1]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44557 44557
   [junit4]   2> 717530 INFO  (Thread-1931) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44557 44557
   [junit4]   2> 717532 WARN  (Thread-1931) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	4	/solr/aliases.json
   [junit4]   2> 	4	/solr/clusterprops.json
   [junit4]   2> 	3	/solr/security.json
   [junit4]   2> 	2	/solr/configs/conf1
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	4	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	4	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 	2	/solr/overseer_elect/election/97384762576797701-127.0.0.1:52338_solr-n_0000000000
   [junit4]   2> 	2	/solr/collections/forwardingCollection/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	8	/solr/overseer/queue
   [junit4]   2> 	4	/solr/collections
   [junit4]   2> 	4	/solr/overseer/collection-queue-work
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 
   [junit4]   2> 717563 INFO  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[4017E5C79AB987F1]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> Feb 01, 2017 6:26:22 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> Feb 01, 2017 6:26:42 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   [junit4]   2>    1) Thread[id=8143, name=jetty-launcher-1642-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2>         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2>         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2>         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2>         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2>    2) Thread[id=8145, name=jetty-launcher-1642-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2>         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2>         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2>         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2>         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> Feb 01, 2017 6:26:42 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=8143, name=jetty-launcher-1642-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2>    2) Thread[id=8145, name=jetty-launcher-1642-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2> 737809 ERROR (jetty-launcher-1642-thread-1-EventThread) [n:127.0.0.1:52338_solr    ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2> 	at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2> 	at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2> 	at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2> 	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 737810 ERROR (jetty-launcher-1642-thread-2-EventThread) [n:127.0.0.1:49782_solr    ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2> 	at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2> 	at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2> 	at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2> 	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 737811 WARN  (jetty-launcher-1642-thread-2-EventThread) [n:127.0.0.1:49782_solr    ] o.a.c.ConnectionState Session expired event received
   [junit4]   2> Feb 01, 2017 6:26:42 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> 737812 WARN  (jetty-launcher-1642-thread-2-SendThread(127.0.0.1:44557)) [n:127.0.0.1:49782_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   [junit4]   2> NOTE: test params are: codec=HighCompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=17533, maxDocsPerChunk=6, blockSize=414), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=17533, blockSize=414)), sim=RandomSimilarity(queryNorm=false): {}, locale=sl, timezone=Pacific/Tongatapu
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=3,free=137061608,total=357896192
   [junit4]   2> NOTE: All tests run in this JVM: [FullSolrCloudDistribCmdsTest, TestJmxMonitoredMap, DirectUpdateHandlerOptimizeTest, TestWriterPerf, SolrCoreMetricManagerTest, CSVRequestHandlerTest, TestPerFieldSimilarityWithDefaultOverride, TestJettySolrRunner, TestDocTermOrds, BasicDistributedZk2Test, CloneFieldUpdateProcessorFactoryTest, TestStressUserVersions, SortSpecParsingTest, TestSSLRandomization, TestCrossCoreJoin, ShowFileRequestHandlerTest, SharedFSAutoReplicaFailoverTest, SuggesterTSTTest, RemoteQueryErrorTest, TestRandomFaceting, TestConfigSetsAPI, ClassificationUpdateProcessorIntegrationTest, TestCoreDiscovery, TestDFISimilarityFactory, ScriptEngineTest, RuleEngineTest, TestCoreAdminApis, SolrMetricReporterTest, SpellPossibilityIteratorTest, TestSimpleQParserPlugin, DocValuesNotIndexedTest, CurrencyFieldOpenExchangeTest, CollectionStateFormat2Test, HdfsNNFailoverTest, TestExactSharedStatsCache, TestMinMaxOnMultiValuedField, TestSolr4Spatial, ShardRoutingCustomTest, SparseHLLTest, TestLRUStatsCache, TlogReplayBufferedWhileIndexingTest, TestHdfsCloudBackupRestore, TestGraphMLResponseWriter, AddBlockUpdateTest, SpellCheckComponentTest, VersionInfoTest, TestPHPSerializedResponseWriter, TestSolrConfigHandlerCloud, SuggestComponentContextFilterQueryTest, SpellCheckCollatorTest, RollingRestartTest, TestSolrXml, TestHdfsBackupRestoreCore, UtilsToolTest, TestLuceneMatchVersion, TestRealTimeGet, CoreAdminRequestStatusTest, CacheHeaderTest, BasicAuthStandaloneTest, TestRecovery, TestExportWriter, HdfsChaosMonkeySafeLeaderTest, TestFieldTypeResource, TestSchemalessBufferedUpdates, RAMDirectoryFactoryTest, SortByFunctionTest, MinimalSchemaTest, BooleanFieldTest, LoggingHandlerTest, HdfsBasicDistributedZkTest, TestLegacyFieldCache, QueryParsingTest, UpdateRequestProcessorFactoryTest, TestComplexPhraseLeadingWildcard, TestSolrCLIRunExample, CloudExitableDirectoryReaderTest, TemplateUpdateProcessorTest, TestDeleteCollectionOnDownNodes, TestHighFrequencyDictionaryFactory, TestManagedSchemaAPI, SimpleCollectionCreateDeleteTest, SecurityConfHandlerTest, TestSolr4Spatial2, AnalyticsQueryTest, CdcrRequestHandlerTest, BlockDirectoryTest, TermVectorComponentDistributedTest, TestSolrFieldCacheMBean, TestDistributedStatsComponentCardinality, TestStressInPlaceUpdates, TestSolrCloudWithSecureImpersonation]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=4017E5C79AB987F1 -Dtests.slow=true -Dtests.locale=sl -Dtests.timezone=Pacific/Tongatapu -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J1 | TestSolrCloudWithSecureImpersonation (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   [junit4]    >    1) Thread[id=8143, name=jetty-launcher-1642-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]    >    2) Thread[id=8145, name=jetty-launcher-1642-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4017E5C79AB987F1]:0)
   [junit4] Completed [190/691 (1!)] on J1 in 52.92s, 12 tests, 1 error <<< FAILURES!

[...truncated 64081 lines...]


[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1110 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1110/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

3 tests failed.
FAILED:  org.apache.solr.core.TestLazyCores.testNoCommit

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([EBF87B31398A7E7D:3498DAE0F2AD1DD8]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:857)
	at org.apache.solr.core.TestLazyCores.check10(TestLazyCores.java:794)
	at org.apache.solr.core.TestLazyCores.testNoCommit(TestLazyCores.java:776)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound='10']
	xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>

<lst name="responseHeader">
  <int name="status">0</int>
  <int name="QTime">0</int>
  <lst name="params">
    <str name="q">*:*</str>
  </lst>
</lst>
<result name="response" numFound="0" start="0">
</result>
</response>

	request was:q=*:*
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:850)
	... 41 more


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores:     1) Thread[id=15367, name=searcherExecutor-7282-thread-1, state=WAITING, group=TGRP-TestLazyCores]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   1) Thread[id=15367, name=searcherExecutor-7282-thread-1, state=WAITING, group=TGRP-TestLazyCores]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([EBF87B31398A7E7D]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=15367, name=searcherExecutor-7282-thread-1, state=WAITING, group=TGRP-TestLazyCores]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=15367, name=searcherExecutor-7282-thread-1, state=WAITING, group=TGRP-TestLazyCores]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([EBF87B31398A7E7D]:0)




Build Log:
[...truncated 11827 lines...]
   [junit4] Suite: org.apache.solr.core.TestLazyCores
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/init-core-data-001
   [junit4]   2> 1875681 INFO  (SUITE-TestLazyCores-seed#[EBF87B31398A7E7D]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields
   [junit4]   2> 1875682 INFO  (SUITE-TestLazyCores-seed#[EBF87B31398A7E7D]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1875683 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testLazySearch
   [junit4]   2> 1875707 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    ] o.a.s.c.SolrCores Allocating transient cache for 4 transient cores
   [junit4]   2> 1875764 WARN  (coreLoadExecutor-7153-thread-2) [    x:collection2] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1875764 WARN  (coreLoadExecutor-7153-thread-1) [    x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1875764 WARN  (coreLoadExecutor-7153-thread-3) [    x:collection5] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1875765 INFO  (coreLoadExecutor-7153-thread-2) [    x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1875765 INFO  (coreLoadExecutor-7153-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1875765 INFO  (coreLoadExecutor-7153-thread-3) [    x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1875773 INFO  (coreLoadExecutor-7153-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
   [junit4]   2> 1875773 INFO  (coreLoadExecutor-7153-thread-3) [    x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
   [junit4]   2> 1875773 INFO  (coreLoadExecutor-7153-thread-2) [    x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
   [junit4]   2> 1875784 INFO  (coreLoadExecutor-7153-thread-3) [    x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1875784 INFO  (coreLoadExecutor-7153-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1875786 INFO  (coreLoadExecutor-7153-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-001/collection1
   [junit4]   2> 1875786 INFO  (coreLoadExecutor-7153-thread-3) [    x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-001/collection5
   [junit4]   2> 1875787 INFO  (coreLoadExecutor-7153-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-001/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-001/collection1/data/]
   [junit4]   2> 1875787 INFO  (coreLoadExecutor-7153-thread-3) [    x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-001/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-001/collection5/data/]
   [junit4]   2> 1875784 INFO  (coreLoadExecutor-7153-thread-2) [    x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1875788 INFO  (coreLoadExecutor-7153-thread-2) [    x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-001/collection2
   [junit4]   2> 1875788 INFO  (coreLoadExecutor-7153-thread-2) [    x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-001/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-001/collection2/data/]
   [junit4]   2> 1875789 INFO  (coreLoadExecutor-7153-thread-3) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1875790 INFO  (coreLoadExecutor-7153-thread-2) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1875791 INFO  (coreLoadExecutor-7153-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1875898 INFO  (coreLoadExecutor-7153-thread-3) [    x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1875898 INFO  (coreLoadExecutor-7153-thread-3) [    x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1875898 INFO  (coreLoadExecutor-7153-thread-3) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=68.1572265625, floorSegmentMB=0.734375, forceMergeDeletesPctAllowed=1.5389211782231416, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2861899137086759
   [junit4]   2> 1875899 INFO  (coreLoadExecutor-7153-thread-3) [    x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@3ab42e0e[collection5] main]
   [junit4]   2> 1875899 INFO  (coreLoadExecutor-7153-thread-3) [    x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-001/collection5/conf
   [junit4]   2> 1875899 INFO  (coreLoadExecutor-7153-thread-3) [    x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1875899 INFO  (searcherExecutor-7155-thread-1-processing-x:collection5) [    x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@3ab42e0e[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1875899 INFO  (coreLoadExecutor-7153-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1875900 INFO  (coreLoadExecutor-7153-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1875900 INFO  (coreLoadExecutor-7153-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=68.1572265625, floorSegmentMB=0.734375, forceMergeDeletesPctAllowed=1.5389211782231416, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2861899137086759
   [junit4]   2> 1875901 INFO  (coreLoadExecutor-7153-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@c58fd60[collection1] main]
   [junit4]   2> 1875901 INFO  (coreLoadExecutor-7153-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-001/collection1/conf
   [junit4]   2> 1875901 INFO  (coreLoadExecutor-7153-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1875901 INFO  (searcherExecutor-7154-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@c58fd60[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1875902 INFO  (coreLoadExecutor-7153-thread-2) [    x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1875902 INFO  (coreLoadExecutor-7153-thread-2) [    x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1875903 INFO  (coreLoadExecutor-7153-thread-2) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=68.1572265625, floorSegmentMB=0.734375, forceMergeDeletesPctAllowed=1.5389211782231416, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2861899137086759
   [junit4]   2> 1875904 INFO  (coreLoadExecutor-7153-thread-2) [    x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@6c54c4ee[collection2] main]
   [junit4]   2> 1875905 INFO  (coreLoadExecutor-7153-thread-2) [    x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-001/collection2/conf
   [junit4]   2> 1875905 INFO  (coreLoadExecutor-7153-thread-2) [    x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1875905 INFO  (searcherExecutor-7156-thread-1-processing-x:collection2) [    x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@6c54c4ee[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1875905 INFO  (coreLoadExecutor-7153-thread-2) [    x:collection2] o.a.s.c.SolrCores Opening transient core collection2
   [junit4]   2> 1875916 WARN  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1875917 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1875923 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.IndexSchema [collection4] Schema name=tiny
   [junit4]   2> 1875926 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1875927 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'collection4' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-001/collection4
   [junit4]   2> 1875928 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrCore [[collection4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-001/collection4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-001/collection4/data/]
   [junit4]   2> 1875930 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=89.7568359375, floorSegmentMB=0.2275390625, forceMergeDeletesPctAllowed=16.58082702084713, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4720026230056362
   [junit4]   2> 1875987 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1875988 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1875988 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=65.267578125, floorSegmentMB=1.16796875, forceMergeDeletesPctAllowed=8.902048251819815, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1875990 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@5235d0e0[collection4] main]
   [junit4]   2> 1875990 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-001/collection4/conf
   [junit4]   2> 1875990 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1875990 INFO  (searcherExecutor-7166-thread-1-processing-x:collection4) [    x:collection4] o.a.s.c.SolrCore [collection4] Registered new searcher Searcher@5235d0e0[collection4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1875991 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1875991 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@4df2fde3
   [junit4]   2> 1876014 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@3e87af8a[collection4] main]
   [junit4]   2> 1876014 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1876014 INFO  (searcherExecutor-7166-thread-1-processing-x:collection4) [    x:collection4] o.a.s.c.SolrCore [collection4] Registered new searcher Searcher@3e87af8a[collection4] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):c7)))}
   [junit4]   2> 1876015 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [collection4]  webapp=null path=null params={q={!prefix+f%3Dv_t}hel&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 1876017 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [collection4]  webapp=null path=null params={q={!raw+f%3Dv_t}hello&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 1876018 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [collection4]  webapp=null path=null params={q={!raw+f%3Dv_t}Hello&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1876018 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [collection4]  webapp=null path=null params={q={!raw+f%3Dv_f}1.5&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1876018 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q={!raw+f%3Dv_t}hello&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1876018 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1792313226
   [junit4]   2> 1876019 INFO  (coreCloseExecutor-7170-thread-1-processing-x:collection4) [    x:collection5] o.a.s.c.SolrCore [collection5]  CLOSING SolrCore org.apache.solr.core.SolrCore@6bd0684c
   [junit4]   2> 1876019 INFO  (coreCloseExecutor-7170-thread-2-processing-x:collection4) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@6436a7be
   [junit4]   2> 1876033 INFO  (coreCloseExecutor-7170-thread-3-processing-x:collection4) [    x:collection4] o.a.s.c.SolrCore [collection4]  CLOSING SolrCore org.apache.solr.core.SolrCore@2c4c9326
   [junit4]   2> 1876043 INFO  (coreCloseExecutor-7170-thread-4-processing-x:collection4) [    x:collection2] o.a.s.c.SolrCore [collection2]  CLOSING SolrCore org.apache.solr.core.SolrCore@2418b52b
   [junit4]   2> 1876044 INFO  (coreCloseExecutor-7170-thread-3-processing-x:collection4) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection4
   [junit4]   2> 1876044 INFO  (coreCloseExecutor-7170-thread-2-processing-x:collection4) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 1876044 INFO  (coreCloseExecutor-7170-thread-1-processing-x:collection4) [    x:collection5] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection5
   [junit4]   2> 1876045 INFO  (coreCloseExecutor-7170-thread-4-processing-x:collection4) [    x:collection2] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection2
   [junit4]   2> 1876045 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1876045 INFO  (TEST-TestLazyCores.testLazySearch-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.SolrTestCaseJ4 ###Ending testLazySearch
   [junit4]   2> 1876046 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.SolrTestCaseJ4 ###Starting testLazyLoad
   [junit4]   2> 1876075 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrCores Allocating transient cache for 4 transient cores
   [junit4]   2> 1876167 WARN  (coreLoadExecutor-7177-thread-1-processing-x:collection4) [    x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1876168 INFO  (coreLoadExecutor-7177-thread-1-processing-x:collection4) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1876170 WARN  (coreLoadExecutor-7177-thread-3-processing-x:collection4) [    x:collection5] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1876171 INFO  (coreLoadExecutor-7177-thread-3-processing-x:collection4) [    x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1876176 INFO  (coreLoadExecutor-7177-thread-1-processing-x:collection4) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
   [junit4]   2> 1876178 INFO  (coreLoadExecutor-7177-thread-3-processing-x:collection4) [    x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
   [junit4]   2> 1876180 INFO  (coreLoadExecutor-7177-thread-3-processing-x:collection4) [    x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1876182 INFO  (coreLoadExecutor-7177-thread-3-processing-x:collection4) [    x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection5
   [junit4]   2> 1876182 INFO  (coreLoadExecutor-7177-thread-3-processing-x:collection4) [    x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection5/data/]
   [junit4]   2> 1876184 INFO  (coreLoadExecutor-7177-thread-3-processing-x:collection4) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1876187 WARN  (coreLoadExecutor-7177-thread-2-processing-x:collection4) [    x:collection2] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1876187 INFO  (coreLoadExecutor-7177-thread-1-processing-x:collection4) [    x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1876188 INFO  (coreLoadExecutor-7177-thread-2-processing-x:collection4) [    x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1876189 INFO  (coreLoadExecutor-7177-thread-1-processing-x:collection4) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection1
   [junit4]   2> 1876190 INFO  (coreLoadExecutor-7177-thread-1-processing-x:collection4) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection1/data/]
   [junit4]   2> 1876191 INFO  (coreLoadExecutor-7177-thread-1-processing-x:collection4) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1876217 INFO  (coreLoadExecutor-7177-thread-2-processing-x:collection4) [    x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
   [junit4]   2> 1876220 INFO  (coreLoadExecutor-7177-thread-2-processing-x:collection4) [    x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1876221 INFO  (coreLoadExecutor-7177-thread-2-processing-x:collection4) [    x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection2
   [junit4]   2> 1876221 INFO  (coreLoadExecutor-7177-thread-2-processing-x:collection4) [    x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection2/data/]
   [junit4]   2> 1876223 INFO  (coreLoadExecutor-7177-thread-2-processing-x:collection4) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1876256 INFO  (coreLoadExecutor-7177-thread-1-processing-x:collection4) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1876256 INFO  (coreLoadExecutor-7177-thread-3-processing-x:collection4) [    x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1876256 INFO  (coreLoadExecutor-7177-thread-3-processing-x:collection4) [    x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1876256 INFO  (coreLoadExecutor-7177-thread-1-processing-x:collection4) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1876257 INFO  (coreLoadExecutor-7177-thread-3-processing-x:collection4) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=68.1572265625, floorSegmentMB=0.734375, forceMergeDeletesPctAllowed=1.5389211782231416, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2861899137086759
   [junit4]   2> 1876257 INFO  (coreLoadExecutor-7177-thread-1-processing-x:collection4) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=68.1572265625, floorSegmentMB=0.734375, forceMergeDeletesPctAllowed=1.5389211782231416, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2861899137086759
   [junit4]   2> 1876258 INFO  (coreLoadExecutor-7177-thread-3-processing-x:collection4) [    x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@749105e3[collection5] main]
   [junit4]   2> 1876258 INFO  (coreLoadExecutor-7177-thread-1-processing-x:collection4) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6204c579[collection1] main]
   [junit4]   2> 1876258 INFO  (coreLoadExecutor-7177-thread-3-processing-x:collection4) [    x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection5/conf
   [junit4]   2> 1876258 INFO  (coreLoadExecutor-7177-thread-1-processing-x:collection4) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection1/conf
   [junit4]   2> 1876259 INFO  (coreLoadExecutor-7177-thread-3-processing-x:collection4) [    x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1876259 INFO  (coreLoadExecutor-7177-thread-1-processing-x:collection4) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1876259 INFO  (searcherExecutor-7179-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6204c579[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1876271 INFO  (coreLoadExecutor-7177-thread-2-processing-x:collection4) [    x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1876271 INFO  (coreLoadExecutor-7177-thread-2-processing-x:collection4) [    x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1876271 INFO  (coreLoadExecutor-7177-thread-2-processing-x:collection4) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=68.1572265625, floorSegmentMB=0.734375, forceMergeDeletesPctAllowed=1.5389211782231416, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2861899137086759
   [junit4]   2> 1876272 INFO  (searcherExecutor-7178-thread-1-processing-x:collection5) [    x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@749105e3[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1876272 INFO  (coreLoadExecutor-7177-thread-2-processing-x:collection4) [    x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@f62ac44[collection2] main]
   [junit4]   2> 1876273 INFO  (coreLoadExecutor-7177-thread-2-processing-x:collection4) [    x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection2/conf
   [junit4]   2> 1876273 INFO  (coreLoadExecutor-7177-thread-2-processing-x:collection4) [    x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1876273 INFO  (searcherExecutor-7180-thread-1-processing-x:collection2) [    x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@f62ac44[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1876273 INFO  (coreLoadExecutor-7177-thread-2-processing-x:collection4) [    x:collection2] o.a.s.c.SolrCores Opening transient core collection2
   [junit4]   2> 1876284 WARN  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1876285 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1876291 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.IndexSchema [collection3] Schema name=tiny
   [junit4]   2> 1876294 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1876295 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'collection3' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection3
   [junit4]   2> 1876296 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrCore [[collection3] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection3], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection3/data/]
   [junit4]   2> 1876298 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=42, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10697595332490756]
   [junit4]   2> 1876348 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1876349 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1876349 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=35, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=28.3359375, floorSegmentMB=1.943359375, forceMergeDeletesPctAllowed=9.93974541185539, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1876350 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@2de34e21[collection3] main]
   [junit4]   2> 1876350 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection3/conf
   [junit4]   2> 1876350 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1876351 INFO  (searcherExecutor-7190-thread-1-processing-x:collection4) [    x:collection4] o.a.s.c.SolrCore [collection3] Registered new searcher Searcher@2de34e21[collection3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1876351 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrCores Opening transient core collection3
   [junit4]   2> 1876371 WARN  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1876371 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1876378 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.IndexSchema [collection4] Schema name=tiny
   [junit4]   2> 1876380 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1876382 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'collection4' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection4
   [junit4]   2> 1876382 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrCore [[collection4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection4/data/]
   [junit4]   2> 1876384 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=34, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.775136014728362]
   [junit4]   2> 1876436 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1876436 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1876437 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=82.6005859375, floorSegmentMB=0.9951171875, forceMergeDeletesPctAllowed=24.019527436928094, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4330838683565321
   [junit4]   2> 1876437 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@278b4edf[collection4] main]
   [junit4]   2> 1876438 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-002/collection4/conf
   [junit4]   2> 1876438 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1876438 INFO  (searcherExecutor-7194-thread-1-processing-x:collection4) [    x:collection4] o.a.s.c.SolrCore [collection4] Registered new searcher Searcher@278b4edf[collection4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1876438 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.CoreContainer Shutting down CoreContainer instance=649893675
   [junit4]   2> 1876439 INFO  (coreCloseExecutor-7198-thread-1-processing-x:collection4) [    x:collection5] o.a.s.c.SolrCore [collection5]  CLOSING SolrCore org.apache.solr.core.SolrCore@7f8baf3
   [junit4]   2> 1876439 INFO  (coreCloseExecutor-7198-thread-2-processing-x:collection4) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@37f08d52
   [junit4]   2> 1876440 INFO  (coreCloseExecutor-7198-thread-3-processing-x:collection4) [    x:collection4] o.a.s.c.SolrCore [collection4]  CLOSING SolrCore org.apache.solr.core.SolrCore@1aeb7a77
   [junit4]   2> 1876453 INFO  (coreCloseExecutor-7198-thread-4-processing-x:collection4) [    x:collection2] o.a.s.c.SolrCore [collection2]  CLOSING SolrCore org.apache.solr.core.SolrCore@3ad8aef2
   [junit4]   2> 1876453 INFO  (coreCloseExecutor-7198-thread-5-processing-x:collection4) [    x:collection3] o.a.s.c.SolrCore [collection3]  CLOSING SolrCore org.apache.solr.core.SolrCore@3a63e596
   [junit4]   2> 1876453 INFO  (coreCloseExecutor-7198-thread-1-processing-x:collection4) [    x:collection5] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection5
   [junit4]   2> 1876454 INFO  (coreCloseExecutor-7198-thread-4-processing-x:collection4) [    x:collection2] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection2
   [junit4]   2> 1876454 INFO  (coreCloseExecutor-7198-thread-2-processing-x:collection4) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 1876454 INFO  (coreCloseExecutor-7198-thread-3-processing-x:collection4) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection4
   [junit4]   2> 1876454 INFO  (coreCloseExecutor-7198-thread-5-processing-x:collection4) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection3
   [junit4]   2> 1876454 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1876455 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.SolrTestCaseJ4 ###Ending testLazyLoad
   [junit4]   2> 1876456 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.SolrTestCaseJ4 ###Starting testBadConfigsGenerateErrors
   [junit4]   2> 1876507 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.CorePropertiesLocator Found 6 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003
   [junit4]   2> 1876508 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.CorePropertiesLocator Cores are: [core1, badConfig2, badSchema1, core2, badConfig1, badSchema2]
   [junit4]   2> 1876518 WARN  (coreLoadExecutor-7205-thread-3-processing-x:collection4) [    x:badSchema1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1876518 WARN  (coreLoadExecutor-7205-thread-1-processing-x:collection4) [    x:core1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1876519 INFO  (coreLoadExecutor-7205-thread-2-processing-x:collection4) [    x:badConfig2] o.a.s.c.Config Ignoring exception matching \QSAXParseException\E
   [junit4]   2> 1876519 INFO  (coreLoadExecutor-7205-thread-3-processing-x:collection4) [    x:badSchema1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1876519 INFO  (coreLoadExecutor-7205-thread-1-processing-x:collection4) [    x:core1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1876556 INFO  (coreLoadExecutor-7205-thread-3-processing-x:collection4) [    x:badSchema1] o.a.s.c.Config Ignoring exception matching \QSAXParseException\E
   [junit4]   2> 1876556 INFO  (coreLoadExecutor-7205-thread-1-processing-x:collection4) [    x:core1] o.a.s.s.IndexSchema [core1] Schema name=tiny
   [junit4]   2> 1876558 INFO  (coreLoadExecutor-7205-thread-3-processing-x:collection4) [    x:badConfig1] o.a.s.c.Config Ignoring exception matching \QSAXParseException\E
   [junit4]   2> 1876559 INFO  (coreLoadExecutor-7205-thread-1-processing-x:collection4) [    x:core1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1876560 INFO  (coreLoadExecutor-7205-thread-1-processing-x:collection4) [    x:core1] o.a.s.c.CoreContainer Creating SolrCore 'core1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/core1
   [junit4]   2> 1876560 INFO  (coreLoadExecutor-7205-thread-1-processing-x:collection4) [    x:core1] o.a.s.c.SolrCore [[core1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/core1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/core1/data/]
   [junit4]   2> 1876563 INFO  (coreLoadExecutor-7205-thread-1-processing-x:collection4) [    x:core1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1876565 WARN  (coreLoadExecutor-7205-thread-2-processing-x:collection4) [    x:core2] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1876566 INFO  (coreLoadExecutor-7205-thread-2-processing-x:collection4) [    x:core2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1876569 WARN  (coreLoadExecutor-7205-thread-3-processing-x:collection4) [    x:badSchema2] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1876570 INFO  (coreLoadExecutor-7205-thread-3-processing-x:collection4) [    x:badSchema2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1876575 INFO  (coreLoadExecutor-7205-thread-2-processing-x:collection4) [    x:core2] o.a.s.s.IndexSchema [core2] Schema name=tiny
   [junit4]   2> 1876592 INFO  (coreLoadExecutor-7205-thread-3-processing-x:collection4) [    x:badSchema2] o.a.s.c.Config Ignoring exception matching \QSAXParseException\E
   [junit4]   2> 1876594 INFO  (coreLoadExecutor-7205-thread-2-processing-x:collection4) [    x:core2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1876595 INFO  (coreLoadExecutor-7205-thread-2-processing-x:collection4) [    x:core2] o.a.s.c.CoreContainer Creating SolrCore 'core2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/core2
   [junit4]   2> 1876596 INFO  (coreLoadExecutor-7205-thread-2-processing-x:collection4) [    x:core2] o.a.s.c.SolrCore [[core2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/core2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/core2/data/]
   [junit4]   2> 1876597 INFO  (coreLoadExecutor-7205-thread-2-processing-x:collection4) [    x:core2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1876646 INFO  (coreLoadExecutor-7205-thread-2-processing-x:collection4) [    x:core2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1876646 INFO  (coreLoadExecutor-7205-thread-2-processing-x:collection4) [    x:core2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1876647 INFO  (coreLoadExecutor-7205-thread-2-processing-x:collection4) [    x:core2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=68.1572265625, floorSegmentMB=0.734375, forceMergeDeletesPctAllowed=1.5389211782231416, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2861899137086759
   [junit4]   2> 1876648 INFO  (coreLoadExecutor-7205-thread-2-processing-x:collection4) [    x:core2] o.a.s.s.SolrIndexSearcher Opening [Searcher@17db68ef[core2] main]
   [junit4]   2> 1876648 INFO  (coreLoadExecutor-7205-thread-2-processing-x:collection4) [    x:core2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/core2/conf
   [junit4]   2> 1876649 INFO  (coreLoadExecutor-7205-thread-2-processing-x:collection4) [    x:core2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1876649 INFO  (searcherExecutor-7207-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrCore [core2] Registered new searcher Searcher@17db68ef[core2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1876650 INFO  (coreLoadExecutor-7205-thread-1-processing-x:collection4) [    x:core1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1876650 INFO  (coreLoadExecutor-7205-thread-1-processing-x:collection4) [    x:core1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1876650 INFO  (coreLoadExecutor-7205-thread-1-processing-x:collection4) [    x:core1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=68.1572265625, floorSegmentMB=0.734375, forceMergeDeletesPctAllowed=1.5389211782231416, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2861899137086759
   [junit4]   2> 1876651 INFO  (coreLoadExecutor-7205-thread-1-processing-x:collection4) [    x:core1] o.a.s.s.SolrIndexSearcher Opening [Searcher@12861988[core1] main]
   [junit4]   2> 1876651 INFO  (coreLoadExecutor-7205-thread-1-processing-x:collection4) [    x:core1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/core1/conf
   [junit4]   2> 1876651 INFO  (coreLoadExecutor-7205-thread-1-processing-x:collection4) [    x:core1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1876651 INFO  (searcherExecutor-7206-thread-1-processing-x:core1) [    x:core1] o.a.s.c.SolrCore [core1] Registered new searcher Searcher@12861988[core1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1876654 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1876655 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@5f923956
   [junit4]   2> 1876672 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@66a38b30[core1] main]
   [junit4]   2> 1876677 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1876677 INFO  (searcherExecutor-7206-thread-1-processing-x:collection4) [    x:collection4] o.a.s.c.SolrCore [core1] Registered new searcher Searcher@66a38b30[core1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):c7)))}
   [junit4]   2> 1876677 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [core1]  webapp=null path=null params={q={!prefix+f%3Dv_t}hel&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 1876678 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [core1]  webapp=null path=null params={q={!raw+f%3Dv_t}hello&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 1876678 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [core1]  webapp=null path=null params={q={!raw+f%3Dv_t}Hello&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1876678 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [core1]  webapp=null path=null params={q={!raw+f%3Dv_f}1.5&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1876695 WARN  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1876695 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1876702 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.IndexSchema [badConfig1] Schema name=tiny
   [junit4]   2> 1876705 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1876706 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'badConfig1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/badConfig1
   [junit4]   2> 1876707 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrCore [[badConfig1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/badConfig1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/badConfig1/data/]
   [junit4]   2> 1876709 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=705326521, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1876773 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1876773 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1876774 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=21, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1876775 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@1048db23[badConfig1] main]
   [junit4]   2> 1876775 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/badConfig1/conf
   [junit4]   2> 1876775 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1876776 INFO  (searcherExecutor-7220-thread-1-processing-x:collection4) [    x:collection4] o.a.s.c.SolrCore [badConfig1] Registered new searcher Searcher@1048db23[badConfig1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1876776 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrCores Opening transient core badConfig1
   [junit4]   2> 1876786 WARN  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1876787 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1876807 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.IndexSchema [badConfig2] Schema name=tiny
   [junit4]   2> 1876809 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1876811 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'badConfig2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/badConfig2
   [junit4]   2> 1876811 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrCore [[badConfig2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/badConfig2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/badConfig2/data/]
   [junit4]   2> 1876813 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=17.9541015625, floorSegmentMB=0.8466796875, forceMergeDeletesPctAllowed=0.16654245986094218, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.40462225382851547
   [junit4]   2> 1876866 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1876866 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1876867 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=2.0703125, noCFSRatio=1.0]
   [junit4]   2> 1876868 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@293aaf04[badConfig2] main]
   [junit4]   2> 1876868 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/badConfig2/conf
   [junit4]   2> 1876868 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1876868 INFO  (searcherExecutor-7224-thread-1-processing-x:collection4) [    x:collection4] o.a.s.c.SolrCore [badConfig2] Registered new searcher Searcher@293aaf04[badConfig2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1876868 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrCores Opening transient core badConfig2
   [junit4]   2> 1876877 WARN  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1876878 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1876884 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.IndexSchema [badSchema1] Schema name=tiny
   [junit4]   2> 1876886 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1876888 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'badSchema1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/badSchema1
   [junit4]   2> 1876889 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrCore [[badSchema1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/badSchema1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/badSchema1/data/]
   [junit4]   2> 1876891 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=10, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1876946 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1876946 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1876947 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1156459016, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1876948 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@5ec19ea0[badSchema1] main]
   [junit4]   2> 1876948 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/badSchema1/conf
   [junit4]   2> 1876949 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1876949 INFO  (searcherExecutor-7228-thread-1-processing-x:collection4) [    x:collection4] o.a.s.c.SolrCore [badSchema1] Registered new searcher Searcher@5ec19ea0[badSchema1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1876949 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrCores Opening transient core badSchema1
   [junit4]   2> 1876959 WARN  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1876960 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1876980 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.IndexSchema [badSchema2] Schema name=tiny
   [junit4]   2> 1876982 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1876984 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'badSchema2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/badSchema2
   [junit4]   2> 1876984 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrCore [[badSchema2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/badSchema2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/badSchema2/data/]
   [junit4]   2> 1876986 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=26.70703125, floorSegmentMB=0.3271484375, forceMergeDeletesPctAllowed=28.596532234919867, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.35121447378345994
   [junit4]   2> 1877040 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1877040 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1877040 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=49, maxMergedSegmentMB=22.7216796875, floorSegmentMB=1.7041015625, forceMergeDeletesPctAllowed=1.7901354002741887, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.17897445399325174
   [junit4]   2> 1877041 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@707e7b55[badSchema2] main]
   [junit4]   2> 1877042 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-003/badSchema2/conf
   [junit4]   2> 1877042 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1877042 INFO  (searcherExecutor-7232-thread-1-processing-x:collection4) [    x:collection4] o.a.s.c.SolrCore [badSchema2] Registered new searcher Searcher@707e7b55[badSchema2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1877042 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrCores Opening transient core badSchema2
   [junit4]   2> 1877046 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1877046 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@5f923956
   [junit4]   2> 1877063 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@61faf667[core1] main]
   [junit4]   2> 1877063 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1877063 INFO  (searcherExecutor-7206-thread-1-processing-x:collection4) [    x:collection4] o.a.s.c.SolrCore [core1] Registered new searcher Searcher@61faf667[core1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.0.0):c7)))}
   [junit4]   2> 1877064 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [core1]  webapp=null path=null params={q={!prefix+f%3Dv_t}hel&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 1877064 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [core1]  webapp=null path=null params={q={!raw+f%3Dv_t}hello&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 1877065 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [core1]  webapp=null path=null params={q={!raw+f%3Dv_t}Hello&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1877065 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [core1]  webapp=null path=null params={q={!raw+f%3Dv_f}1.5&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1877065 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1877066 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@6dc42226
   [junit4]   2> 1877090 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@ac9c99b[badConfig1] main]
   [junit4]   2> 1877090 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1877090 INFO  (searcherExecutor-7220-thread-1-processing-x:collection4) [    x:collection4] o.a.s.c.SolrCore [badConfig1] Registered new searcher Searcher@ac9c99b[badConfig1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):c7)))}
   [junit4]   2> 1877091 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [badConfig1]  webapp=null path=null params={q={!prefix+f%3Dv_t}hel&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 1877091 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [badConfig1]  webapp=null path=null params={q={!raw+f%3Dv_t}hello&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 1877092 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [badConfig1]  webapp=null path=null params={q={!raw+f%3Dv_t}Hello&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1877092 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [badConfig1]  webapp=null path=null params={q={!raw+f%3Dv_f}1.5&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1877093 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1877093 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@30d007d0
   [junit4]   2> 1877111 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@4b029165[badSchema1] main]
   [junit4]   2> 1877111 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1877111 INFO  (searcherExecutor-7228-thread-1-processing-x:collection4) [    x:collection4] o.a.s.c.SolrCore [badSchema1] Registered new searcher Searcher@4b029165[badSchema1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):c7)))}
   [junit4]   2> 1877112 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [badSchema1]  webapp=null path=null params={q={!prefix+f%3Dv_t}hel&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 1877112 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [badSchema1]  webapp=null path=null params={q={!raw+f%3Dv_t}hello&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 1877113 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [badSchema1]  webapp=null path=null params={q={!raw+f%3Dv_t}Hello&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1877113 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.S.Request [badSchema1]  webapp=null path=null params={q={!raw+f%3Dv_f}1.5&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1877113 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1906033975
   [junit4]   2> 1877113 INFO  (coreCloseExecutor-7242-thread-1-processing-x:collection4) [    x:core2] o.a.s.c.SolrCore [core2]  CLOSING SolrCore org.apache.solr.core.SolrCore@34bb2f2d
   [junit4]   2> 1877113 INFO  (coreCloseExecutor-7242-thread-2-processing-x:collection4) [    x:core1] o.a.s.c.SolrCore [core1]  CLOSING SolrCore org.apache.solr.core.SolrCore@4a15b531
   [junit4]   2> 1877128 INFO  (coreCloseExecutor-7242-thread-3-processing-x:collection4) [    x:badConfig1] o.a.s.c.SolrCore [badConfig1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1d1d739d
   [junit4]   2> 1877128 INFO  (coreCloseExecutor-7242-thread-4-processing-x:collection4) [    x:badConfig2] o.a.s.c.SolrCore [badConfig2]  CLOSING SolrCore org.apache.solr.core.SolrCore@4e3cd5a6
   [junit4]   2> 1877128 INFO  (coreCloseExecutor-7242-thread-2-processing-x:collection4) [    x:core1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.core1
   [junit4]   2> 1877129 INFO  (coreCloseExecutor-7242-thread-5-processing-x:collection4) [    x:badSchema1] o.a.s.c.SolrCore [badSchema1]  CLOSING SolrCore org.apache.solr.core.SolrCore@3424a160
   [junit4]   2> 1877129 INFO  (coreCloseExecutor-7242-thread-6-processing-x:collection4) [    x:badSchema2] o.a.s.c.SolrCore [badSchema2]  CLOSING SolrCore org.apache.solr.core.SolrCore@494f5378
   [junit4]   2> 1877129 INFO  (coreCloseExecutor-7242-thread-1-processing-x:collection4) [    x:core2] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.core2
   [junit4]   2> 1877144 INFO  (coreCloseExecutor-7242-thread-5-processing-x:collection4) [    x:badSchema1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.badSchema1
   [junit4]   2> 1877144 INFO  (coreCloseExecutor-7242-thread-6-processing-x:collection4) [    x:badSchema2] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.badSchema2
   [junit4]   2> 1877144 INFO  (coreCloseExecutor-7242-thread-3-processing-x:collection4) [    x:badConfig1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.badConfig1
   [junit4]   2> 1877144 INFO  (coreCloseExecutor-7242-thread-4-processing-x:collection4) [    x:badConfig2] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.badConfig2
   [junit4]   2> 1877145 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1877145 INFO  (TEST-TestLazyCores.testBadConfigsGenerateErrors-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.SolrTestCaseJ4 ###Ending testBadConfigsGenerateErrors
   [junit4]   2> 1877146 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.SolrTestCaseJ4 ###Starting testNoCommit
   [junit4]   2> 1877184 INFO  (TEST-TestLazyCores.testNoCommit-seed#[EBF87B31398A7E7D]) [    x:collection4] o.a.s.c.SolrCores Allocating transient cache for 4 transient cores
   [junit4]   2> 1877255 WARN  (coreLoadExecutor-7249-thread-1-processing-x:collection4) [    x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1877255 WARN  (coreLoadExecutor-7249-thread-2-processing-x:collection4) [    x:collection2] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1877256 INFO  (coreLoadExecutor-7249-thread-2-processing-x:collection4) [    x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1877256 INFO  (coreLoadExecutor-7249-thread-1-processing-x:collection4) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1877289 WARN  (coreLoadExecutor-7249-thread-3-processing-x:collection4) [    x:collection5] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1877290 INFO  (coreLoadExecutor-7249-thread-3-processing-x:collection4) [    x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1877290 INFO  (coreLoadExecutor-7249-thread-1-processing-x:collection4) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
   [junit4]   2> 1877290 INFO  (coreLoadExecutor-7249-thread-2-processing-x:collection4) [    x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
   [junit4]   2> 1877293 INFO  (coreLoadExecutor-7249-thread-1-processing-x:collection4) [    x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1877293 INFO  (coreLoadExecutor-7249-thread-2-processing-x:collection4) [    x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 1877296 INFO  (coreLoadExecutor-7249-thread-2-processing-x:collection4) [    x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_EBF87B31398A7E7D-001/tempDir-004/collection2
   [junit4] 

[...truncated too long message...]

e.solr.core.TestLazyCores: 
   [junit4]   2>    1) Thread[id=15367, name=searcherExecutor-7282-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Feb 01, 2017 11:07:25 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=15367, name=searcherExecutor-7282-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2> Feb 01, 2017 11:07:28 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=15367, name=searcherExecutor-7282-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {v_t=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), v_s=PostingsFormat(name=Memory), v_f=Lucene50(blocksize=128)}, docValues:{}, maxPointsInLeafNode=1433, maxMBSortInHeap=5.394887927934424, sim=RandomSimilarity(queryNorm=true): {}, locale=lt-LT, timezone=America/Matamoros
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=2,free=151285632,total=500998144
   [junit4]   2> NOTE: All tests run in this JVM: [ClassificationUpdateProcessorTest, TestGraphTermsQParserPlugin, CopyFieldTest, TestSolrQueryResponse, TestCopyFieldCollectionResource, TestSizeLimitedDistributedMap, DOMUtilTest, SegmentsInfoRequestHandlerTest, TestSurroundQueryParser, TestEmbeddedSolrServerConstructors, TestPerFieldSimilarity, RecoveryZkTest, DistributedVersionInfoTest, DeleteNodeTest, TestComplexPhraseQParserPlugin, TestTestInjection, RulesTest, NumericFieldsTest, TestRTimerTree, ReplaceNodeTest, LeaderElectionContextKeyTest, JsonValidatorTest, ForceLeaderTest, PingRequestHandlerTest, TestNamedUpdateProcessors, TestPostingsSolrHighlighter, TestQueryWrapperFilter, PKIAuthenticationIntegrationTest, TestJavabinTupleStreamParser, ManagedSchemaRoundRobinCloudTest, ChaosMonkeyNothingIsSafeTest, TestDistribDocBasedVersion, SOLR749Test, TestBlendedInfixSuggestions, TestIBSimilarityFactory, RankQueryTest, DistributedDebugComponentTest, RequestLoggingTest, PolyFieldTest, CoreSorterTest, OverseerTest, TestWordDelimiterFilterFactory, DistributedFacetPivotLongTailTest, HdfsRecoverLeaseTest, CurrencyFieldXmlFileTest, TestReplicationHandler, QueryEqualityTest, ExitableDirectoryReaderTest, DistributedFacetExistsSmallTest, TestBM25SimilarityFactory, TestTolerantSearch, TestFreeTextSuggestions, AnalysisErrorHandlingTest, TestSmileRequest, OpenExchangeRatesOrgProviderTest, LeaderFailureAfterFreshStartTest, SuggestComponentTest, PluginInfoTest, TestRandomRequestDistribution, TestMergePolicyConfig, HdfsLockFactoryTest, SSLMigrationTest, AnalysisAfterCoreReloadTest, TestConfigsApi, TestScoreJoinQPScore, TestManagedSchemaThreadSafety, OutOfBoxZkACLAndCredentialsProvidersTest, SpatialFilterTest, MBeansHandlerTest, TestSolrConfigHandlerConcurrent, TestSolrDynamicMBean, TestShardHandlerFactory, TestRestoreCore, SoftAutoCommitTest, TestConfigOverlay, SimpleFacetsTest, TestNonDefinedSimilarityFactory, TestJmxIntegration, HdfsChaosMonkeyNothingIsSafeTest, QueryResultKeyTest, TestSegmentSorting, SolrXmlInZkTest, BitVectorTest, TestCustomDocTransformer, TestJsonFacets, TestCSVResponseWriter, RequiredFieldsTest, DocumentAnalysisRequestHandlerTest, TestManagedSchema, TestXmlQParserPlugin, HighlighterTest, SolrIndexMetricsTest, FullHLLTest, TriLevelCompositeIdRoutingTest, TestRemoteStreaming, TestStressLucene, OutputWriterTest, HdfsThreadLeakTest, SolrMetricManagerTest, TestExclusionRuleCollectionAccess, DisMaxRequestHandlerTest, TestTrackingShardHandlerFactory, PathHierarchyTokenizerFactoryTest, TestFastLRUCache, SolrCoreTest, CoreMergeIndexesAdminHandlerTest, TestNumericTerms32, BadCopyFieldTest, TestNoOpRegenerator, QueryElevationComponentTest, SynonymTokenizerTest, SimplePostToolTest, SolrGraphiteReporterTest, TestTrieFacet, PrimUtilsTest, TestCryptoKeys, SearchHandlerTest, TestSolrDeletionPolicy1, DistributedMLTComponentTest, TestCloudRecovery, HttpSolrCallGetCoreTest, SolrRequestParserTest, TestHighlightDedupGrouping, TestRawResponseWriter, CdcrVersionReplicationTest, TestExceedMaxTermLength, HardAutoCommitTest, TestSweetSpotSimilarityFactory, DirectUpdateHandlerTest, ReplicaListTransformerTest, TestConfigReload, TestConfigSetProperties, AsyncCallRequestStatusResponseTest, TestFieldCacheWithThreads, TestStressCloudBlindAtomicUpdates, TestCursorMarkWithoutUniqueKey, TestHashPartitioner, TestTolerantUpdateProcessorRandomCloud, TestSolrCoreParser, TestFunctionQuery, BlockCacheTest, BufferStoreTest, TestManagedStopFilterFactory, TestFieldCacheSanityChecker, ConjunctionSolrSpellCheckerTest, RecoveryAfterSoftCommitTest, TestInfoStreamLogging, SampleTest, FastVectorHighlighterTest, DocExpirationUpdateProcessorFactoryTest, ParsingFieldUpdateProcessorsTest, TestIntervalFaceting, TestCollationField, TestLeaderElectionWithEmptyReplica, ShardSplitTest, TestRTGBase, MetricsHandlerTest, HdfsBasicDistributedZk2Test, TestCoreContainer, OverseerRolesTest, DistributedQueryComponentCustomSortTest, TestSearcherReuse, TestLockTree, TestSolrCloudSnapshots, TestInPlaceUpdatesDistrib, TestLRUCache, TestStressReorder, SyncSliceTest, ClusterStateUpdateTest, TestRandomFaceting, TestRandomDVFaceting, AutoCommitTest, TestFaceting, DistributedSpellCheckComponentTest, ZkControllerTest, TestReload, TestStressVersions, TestSolr4Spatial, TestGroupingSearch, SolrCmdDistributorTest, PeerSyncTest, ConvertedLegacyTest, TestSort, BasicFunctionalityTest, TestLazyCores]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLazyCores -Dtests.seed=EBF87B31398A7E7D -Dtests.slow=true -Dtests.locale=lt-LT -Dtests.timezone=America/Matamoros -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J0 | TestLazyCores (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   [junit4]    >    1) Thread[id=15367, name=searcherExecutor-7282-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EBF87B31398A7E7D]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=15367, name=searcherExecutor-7282-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EBF87B31398A7E7D]:0)
   [junit4] Completed [378/691 (1!)] on J0 in 32.45s, 9 tests, 3 errors <<< FAILURES!

[...truncated 63496 lines...]


[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1109 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1109/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC

4 tests failed.
FAILED:  org.apache.solr.cloud.PeerSyncReplicationTest.test

Error Message:
timeout waiting to see all nodes active

Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
	at __randomizedtesting.SeedInfo.seed([C9E5AF64279550E3:41B190BE89693D1B]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
	at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
	at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
	at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.core.TestLazyCores.testNoCommit

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([C9E5AF64279550E3:16850EB5ECB23346]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:857)
	at org.apache.solr.core.TestLazyCores.check10(TestLazyCores.java:794)
	at org.apache.solr.core.TestLazyCores.testNoCommit(TestLazyCores.java:776)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound='10']
	xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>

<lst name="responseHeader">
  <int name="status">0</int>
  <int name="QTime">0</int>
  <lst name="params">
    <str name="q">*:*</str>
  </lst>
</lst>
<result name="response" numFound="0" start="0">
</result>
</response>

	request was:q=*:*
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:850)
	... 41 more


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores:     1) Thread[id=7106, name=searcherExecutor-3197-thread-1, state=WAITING, group=TGRP-TestLazyCores]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   1) Thread[id=7106, name=searcherExecutor-3197-thread-1, state=WAITING, group=TGRP-TestLazyCores]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([C9E5AF64279550E3]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=7106, name=searcherExecutor-3197-thread-1, state=WAITING, group=TGRP-TestLazyCores]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=7106, name=searcherExecutor-3197-thread-1, state=WAITING, group=TGRP-TestLazyCores]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([C9E5AF64279550E3]:0)




Build Log:
[...truncated 11082 lines...]
   [junit4] Suite: org.apache.solr.cloud.PeerSyncReplicationTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/init-core-data-001
   [junit4]   2> 637685 INFO  (SUITE-PeerSyncReplicationTest-seed#[C9E5AF64279550E3]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields
   [junit4]   2> 637694 INFO  (SUITE-PeerSyncReplicationTest-seed#[C9E5AF64279550E3]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 637694 INFO  (SUITE-PeerSyncReplicationTest-seed#[C9E5AF64279550E3]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /_hoe/u
   [junit4]   2> 637696 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 637696 INFO  (Thread-3197) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 637696 INFO  (Thread-3197) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 637796 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.ZkTestServer start zk server on port:54049
   [junit4]   2> 637817 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 637821 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 637824 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 637828 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 637831 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 637833 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 637837 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 637840 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 637843 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 637845 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 637848 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 638054 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/control-001/cores/collection1
   [junit4]   2> 638057 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 638059 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@57f1f99a{/_hoe/u,null,AVAILABLE}
   [junit4]   2> 638059 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3831e657{HTTP/1.1,[http/1.1]}{127.0.0.1:37108}
   [junit4]   2> 638059 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.e.j.s.Server Started @641476ms
   [junit4]   2> 638059 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/tempDir-001/control/data, hostContext=/_hoe/u, hostPort=37108, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/control-001/cores}
   [junit4]   2> 638060 ERROR (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 638060 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 638060 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 638060 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 638060 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-01T00:19:02.453Z
   [junit4]   2> 638064 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 638064 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/control-001/solr.xml
   [junit4]   2> 638073 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54049/solr
   [junit4]   2> 638112 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [n:127.0.0.1:37108__hoe%2Fu    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37108__hoe%2Fu
   [junit4]   2> 638114 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [n:127.0.0.1:37108__hoe%2Fu    ] o.a.s.c.Overseer Overseer (id=97380489107275780-127.0.0.1:37108__hoe%2Fu-n_0000000000) starting
   [junit4]   2> 638126 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [n:127.0.0.1:37108__hoe%2Fu    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37108__hoe%2Fu
   [junit4]   2> 638130 INFO  (zkCallback-1561-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 638188 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [n:127.0.0.1:37108__hoe%2Fu    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/control-001/cores
   [junit4]   2> 638188 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [n:127.0.0.1:37108__hoe%2Fu    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 638191 INFO  (OverseerStateUpdate-97380489107275780-127.0.0.1:37108__hoe%2Fu-n_0000000000) [n:127.0.0.1:37108__hoe%2Fu    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 639224 WARN  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 639225 INFO  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 639240 INFO  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 639324 WARN  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 639327 INFO  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 639343 INFO  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 639350 INFO  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/control-001/cores/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/control-001/cores/collection1/data/]
   [junit4]   2> 639350 INFO  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@695c53b9
   [junit4]   2> 639352 INFO  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=23, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 639397 WARN  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 639441 INFO  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 639441 INFO  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 639442 INFO  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 639442 INFO  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 639442 INFO  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=21, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7184834743525046]
   [junit4]   2> 639443 INFO  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@76ad7dc9[collection1] main]
   [junit4]   2> 639444 INFO  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 639445 INFO  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 639445 INFO  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 639446 INFO  (searcherExecutor-2699-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu x:collection1 c:control_collection) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@76ad7dc9[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 639447 INFO  (coreLoadExecutor-2698-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu c:control_collection   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1558087827550371840
   [junit4]   2> 639457 INFO  (coreZkRegister-2691-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu x:collection1 c:control_collection) [n:127.0.0.1:37108__hoe%2Fu c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 639457 INFO  (coreZkRegister-2691-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu x:collection1 c:control_collection) [n:127.0.0.1:37108__hoe%2Fu c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 639457 INFO  (coreZkRegister-2691-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu x:collection1 c:control_collection) [n:127.0.0.1:37108__hoe%2Fu c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:37108/_hoe/u/collection1/
   [junit4]   2> 639457 INFO  (coreZkRegister-2691-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu x:collection1 c:control_collection) [n:127.0.0.1:37108__hoe%2Fu c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 639457 INFO  (coreZkRegister-2691-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu x:collection1 c:control_collection) [n:127.0.0.1:37108__hoe%2Fu c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:37108/_hoe/u/collection1/ has no replicas
   [junit4]   2> 639463 INFO  (coreZkRegister-2691-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu x:collection1 c:control_collection) [n:127.0.0.1:37108__hoe%2Fu c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:37108/_hoe/u/collection1/ shard1
   [junit4]   2> 639614 INFO  (coreZkRegister-2691-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu x:collection1 c:control_collection) [n:127.0.0.1:37108__hoe%2Fu c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 639785 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 639787 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:54049/solr ready
   [junit4]   2> 639787 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 639787 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
   [junit4]   2> 639937 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-1-001/cores/collection1
   [junit4]   2> 639938 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-1-001
   [junit4]   2> 639939 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 639940 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@276253c8{/_hoe/u,null,AVAILABLE}
   [junit4]   2> 639940 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@73decd3a{HTTP/1.1,[http/1.1]}{127.0.0.1:53173}
   [junit4]   2> 639940 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.e.j.s.Server Started @643358ms
   [junit4]   2> 639940 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/_hoe/u, hostPort=53173, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-1-001/cores}
   [junit4]   2> 639940 ERROR (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 639940 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 639941 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 639941 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 639941 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-01T00:19:04.334Z
   [junit4]   2> 639956 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 639956 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-1-001/solr.xml
   [junit4]   2> 639963 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54049/solr
   [junit4]   2> 639980 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 639987 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53173__hoe%2Fu
   [junit4]   2> 639990 INFO  (zkCallback-1565-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 639991 INFO  (zkCallback-1561-thread-4-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 640014 INFO  (zkCallback-1571-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 640089 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-1-001/cores
   [junit4]   2> 640090 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 640095 INFO  (OverseerStateUpdate-97380489107275780-127.0.0.1:37108__hoe%2Fu-n_0000000000) [n:127.0.0.1:37108__hoe%2Fu    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 640201 INFO  (zkCallback-1571-thread-2-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 641113 WARN  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 641114 INFO  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 641127 INFO  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 641212 WARN  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 641214 INFO  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 641236 INFO  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 641237 INFO  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-1-001/cores/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-1-001/cores/collection1/data/]
   [junit4]   2> 641238 INFO  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@695c53b9
   [junit4]   2> 641241 INFO  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=23, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 641250 WARN  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 641296 INFO  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 641296 INFO  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 641297 INFO  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 641297 INFO  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 641298 INFO  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=21, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7184834743525046]
   [junit4]   2> 641299 INFO  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4d9e6ab8[collection1] main]
   [junit4]   2> 641301 INFO  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 641301 INFO  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 641301 INFO  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 641303 INFO  (searcherExecutor-2710-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu x:collection1 c:collection1) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4d9e6ab8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 641304 INFO  (coreLoadExecutor-2709-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu c:collection1   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1558087829497577472
   [junit4]   2> 641315 INFO  (coreZkRegister-2704-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu x:collection1 c:collection1) [n:127.0.0.1:53173__hoe%2Fu c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 641316 INFO  (coreZkRegister-2704-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu x:collection1 c:collection1) [n:127.0.0.1:53173__hoe%2Fu c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 641316 INFO  (coreZkRegister-2704-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu x:collection1 c:collection1) [n:127.0.0.1:53173__hoe%2Fu c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53173/_hoe/u/collection1/
   [junit4]   2> 641316 INFO  (coreZkRegister-2704-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu x:collection1 c:collection1) [n:127.0.0.1:53173__hoe%2Fu c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 641316 INFO  (coreZkRegister-2704-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu x:collection1 c:collection1) [n:127.0.0.1:53173__hoe%2Fu c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:53173/_hoe/u/collection1/ has no replicas
   [junit4]   2> 641323 INFO  (coreZkRegister-2704-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu x:collection1 c:collection1) [n:127.0.0.1:53173__hoe%2Fu c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53173/_hoe/u/collection1/ shard1
   [junit4]   2> 641429 INFO  (zkCallback-1571-thread-2-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 641485 INFO  (coreZkRegister-2704-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu x:collection1 c:collection1) [n:127.0.0.1:53173__hoe%2Fu c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 641593 INFO  (zkCallback-1571-thread-2-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 641920 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-2-001/cores/collection1
   [junit4]   2> 641922 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-2-001
   [junit4]   2> 641923 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 641924 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6fa0689b{/_hoe/u,null,AVAILABLE}
   [junit4]   2> 641925 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@50470561{HTTP/1.1,[http/1.1]}{127.0.0.1:61596}
   [junit4]   2> 641925 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.e.j.s.Server Started @645344ms
   [junit4]   2> 641925 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/_hoe/u, hostPort=61596, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-2-001/cores}
   [junit4]   2> 641925 ERROR (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 641925 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 641925 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 641925 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 641925 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-01T00:19:06.318Z
   [junit4]   2> 641932 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 641932 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-2-001/solr.xml
   [junit4]   2> 641951 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54049/solr
   [junit4]   2> 641969 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [n:127.0.0.1:61596__hoe%2Fu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 641978 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [n:127.0.0.1:61596__hoe%2Fu    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:61596__hoe%2Fu
   [junit4]   2> 641980 INFO  (zkCallback-1565-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 641980 INFO  (zkCallback-1571-thread-2-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 641981 INFO  (zkCallback-1561-thread-1-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 642007 INFO  (zkCallback-1577-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 642107 INFO  (zkCallback-1571-thread-2-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 642941 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [n:127.0.0.1:61596__hoe%2Fu    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-2-001/cores
   [junit4]   2> 642941 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [n:127.0.0.1:61596__hoe%2Fu    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 642946 INFO  (OverseerStateUpdate-97380489107275780-127.0.0.1:37108__hoe%2Fu-n_0000000000) [n:127.0.0.1:37108__hoe%2Fu    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 643051 INFO  (zkCallback-1571-thread-2-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 643051 INFO  (zkCallback-1577-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 643982 WARN  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 643983 INFO  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 644005 INFO  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 644094 WARN  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 644096 INFO  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 644115 INFO  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 644115 INFO  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-2-001/cores/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-2-001/cores/collection1/data/]
   [junit4]   2> 644116 INFO  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@695c53b9
   [junit4]   2> 644117 INFO  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=23, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 644124 WARN  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 644172 INFO  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 644172 INFO  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 644172 INFO  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 644173 INFO  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 644174 INFO  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=21, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7184834743525046]
   [junit4]   2> 644175 INFO  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@705b56bd[collection1] main]
   [junit4]   2> 644177 INFO  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 644178 INFO  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 644178 INFO  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 644179 INFO  (searcherExecutor-2721-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 c:collection1) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@705b56bd[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 644179 INFO  (coreLoadExecutor-2720-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu c:collection1   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1558087832512233472
   [junit4]   2> 644185 INFO  (coreZkRegister-2715-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 c:collection1) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
   [junit4]   2> 644186 INFO  (updateExecutor-1574-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 644186 INFO  (recoveryExecutor-1575-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 644187 INFO  (recoveryExecutor-1575-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 644187 INFO  (recoveryExecutor-1575-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
   [junit4]   2> 644187 INFO  (recoveryExecutor-1575-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 644188 INFO  (recoveryExecutor-1575-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:53173/_hoe/u/collection1/] and I am [http://127.0.0.1:61596/_hoe/u/collection1/]
   [junit4]   2> 644193 INFO  (recoveryExecutor-1575-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:53173/_hoe/u]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:61596__hoe%252Fu&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 644196 INFO  (qtp1010960629-8776) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
   [junit4]   2> 644197 INFO  (qtp1010960629-8776) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
   [junit4]   2> 644197 INFO  (qtp1010960629-8776) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:61596__hoe%2Fu, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:61596/_hoe/u","node_name":"127.0.0.1:61596__hoe%2Fu","state":"down"}
   [junit4]   2> 644294 INFO  (zkCallback-1571-thread-2-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 644294 INFO  (zkCallback-1577-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 644790 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-3-001/cores/collection1
   [junit4]   2> 644791 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-3-001
   [junit4]   2> 644792 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 644793 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@42618e52{/_hoe/u,null,AVAILABLE}
   [junit4]   2> 644794 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4e82b1d1{HTTP/1.1,[http/1.1]}{127.0.0.1:63239}
   [junit4]   2> 644794 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.e.j.s.Server Started @648214ms
   [junit4]   2> 644794 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/_hoe/u, hostPort=63239, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-3-001/cores}
   [junit4]   2> 644794 ERROR (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 644795 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 644795 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 644795 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 644795 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-01T00:19:09.188Z
   [junit4]   2> 644800 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 644800 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-3-001/solr.xml
   [junit4]   2> 644807 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54049/solr
   [junit4]   2> 644824 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [n:127.0.0.1:63239__hoe%2Fu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 644832 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [n:127.0.0.1:63239__hoe%2Fu    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63239__hoe%2Fu
   [junit4]   2> 644835 INFO  (zkCallback-1565-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 644836 INFO  (zkCallback-1561-thread-4-processing-n:127.0.0.1:37108__hoe%2Fu) [n:127.0.0.1:37108__hoe%2Fu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 644836 INFO  (zkCallback-1577-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 644836 INFO  (zkCallback-1571-thread-2-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 644839 INFO  (zkCallback-1584-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 644938 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [n:127.0.0.1:63239__hoe%2Fu    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-3-001/cores
   [junit4]   2> 644939 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [n:127.0.0.1:63239__hoe%2Fu    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 644942 INFO  (OverseerStateUpdate-97380489107275780-127.0.0.1:37108__hoe%2Fu-n_0000000000) [n:127.0.0.1:37108__hoe%2Fu    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 645046 INFO  (zkCallback-1577-thread-2-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 645046 INFO  (zkCallback-1571-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 645046 INFO  (zkCallback-1584-thread-2-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 645198 INFO  (qtp1010960629-8776) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:61596__hoe%2Fu, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:61596/_hoe/u","node_name":"127.0.0.1:61596__hoe%2Fu","state":"recovering"}
   [junit4]   2> 645198 INFO  (qtp1010960629-8776) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
   [junit4]   2> 645198 INFO  (qtp1010960629-8776) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:61596__hoe%252Fu&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1002
   [junit4]   2> 645964 WARN  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 645965 INFO  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 645984 INFO  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 646092 WARN  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 646094 INFO  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 646124 INFO  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 646124 INFO  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-3-001/cores/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C9E5AF64279550E3-001/shard-3-001/cores/collection1/data/]
   [junit4]   2> 646125 INFO  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@695c53b9
   [junit4]   2> 646126 INFO  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=23, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 646133 WARN  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 646192 INFO  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 646192 INFO  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 646193 INFO  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 646193 INFO  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 646194 INFO  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=21, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7184834743525046]
   [junit4]   2> 646195 INFO  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1d54d476[collection1] main]
   [junit4]   2> 646197 INFO  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 646197 INFO  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 646197 INFO  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 646201 INFO  (searcherExecutor-2732-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu x:collection1 c:collection1) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@1d54d476[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 646201 INFO  (coreLoadExecutor-2731-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu c:collection1   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1558087834632454144
   [junit4]   2> 646209 INFO  (coreZkRegister-2726-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu x:collection1 c:collection1) [n:127.0.0.1:63239__hoe%2Fu c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
   [junit4]   2> 646209 INFO  (updateExecutor-1581-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:63239__hoe%2Fu c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 646210 INFO  (recoveryExecutor-1582-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:63239__hoe%2Fu c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 646211 INFO  (recoveryExecutor-1582-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:63239__hoe%2Fu c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 646211 INFO  (recoveryExecutor-1582-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:63239__hoe%2Fu c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
   [junit4]   2> 646211 INFO  (recoveryExecutor-1582-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:63239__hoe%2Fu c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 646211 INFO  (recoveryExecutor-1582-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:63239__hoe%2Fu c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:53173/_hoe/u/collection1/] and I am [http://127.0.0.1:63239/_hoe/u/collection1/]
   [junit4]   2> 646216 INFO  (recoveryExecutor-1582-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:63239__hoe%2Fu c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:53173/_hoe/u]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:63239__hoe%252Fu&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 646218 INFO  (qtp1010960629-8765) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
   [junit4]   2> 646219 INFO  (qtp1010960629-8765) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
   [junit4]   2> 646219 INFO  (qtp1010960629-8765) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:63239__hoe%2Fu, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:63239/_hoe/u","node_name":"127.0.0.1:63239__hoe%2Fu","state":"down"}
   [junit4]   2> 646324 INFO  (zkCallback-1571-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 646324 INFO  (zkCallback-1584-thread-2-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 646325 INFO  (zkCallback-1577-thread-2-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 646543 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 646543 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 30 for each attempt
   [junit4]   2> 646543 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C9E5AF64279550E3]) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
   [junit4]   2> 647222 INFO  (qtp1010960629-8765) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:63239__hoe%2Fu, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:63239/_hoe/u","node_name":"127.0.0.1:63239__hoe%2Fu","state":"recovering"}
   [junit4]   2> 647222 INFO  (qtp1010960629-8765) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
   [junit4]   2> 647222 INFO  (qtp1010960629-8765) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:63239__hoe%252Fu&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1004
   [junit4]   2> 652202 INFO  (recoveryExecutor-1575-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:53173/_hoe/u/collection1/] - recoveringAfterStartup=[true]
   [junit4]   2> 652202 INFO  (recoveryExecutor-1575-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:61596/_hoe/u START replicas=[http://127.0.0.1:53173/_hoe/u/collection1/] nUpdates=1000
   [junit4]   2> 652207 INFO  (qtp1010960629-8766) [n:127.0.0.1:53173__hoe%2Fu c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
   [junit4]   2> 652207 INFO  (qtp1010960629-8766) [n:127.0.0.1:53173__hoe%2Fu c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1]  webapp=/_hoe/u path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 652209 INFO  (recoveryExecutor-1575-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
   [junit4]   2> 652209 INFO  (recoveryExecutor-1575-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync 
   [junit4]   2> 652209 INFO  (recoveryExecutor-1575-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 652209 INFO  (recoveryExecutor-1575-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 652210 INFO  (recoveryExecutor-1575-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 652210 INFO  (recoveryExecutor-1575-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
   [junit4]   2> 652210 INFO  (recoveryExecutor-1575-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
   [junit4]   2> 652210 INFO  (recoveryExecutor-1575-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
   [junit4]   2> 652210 INFO  (recoveryExecutor-1575-thread-1-processing-n:127.0.0.1:61596__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:61596__hoe%2Fu c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
   [junit4]   2> 652215 INFO  (zkCallback-1571-thread-1-processing-n:127.0.0.1:53173__hoe%2Fu) [n:127.0.0.1:53173__hoe%2Fu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 652215 INFO  (zkCallback-1584-thread-2-processing-n:127.0.0.1:63239__hoe%2Fu) [n:127.0.0.1:63239__hoe%2Fu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 652215 INFO  (zkCallback-1577-thread-2-processing-n:127.0.0.1:61596__hoe%2Fu) [n:127.0.0.1:61596__hoe%2Fu    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 654226 INFO  (recoveryExecutor-1582-thread-1-processing-n:127.0.0.1:63239__hoe%2Fu x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:63239__hoe%2Fu c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting 

[...truncated too long message...]

eporters for: solr.core.collection5
   [junit4]   2> 1007607 INFO  (coreCloseExecutor-3441-thread-5-processing-x:badConfig1) [    x:collection7] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection7
   [junit4]   2> 1007607 INFO  (coreCloseExecutor-3441-thread-4-processing-x:badConfig1) [    x:collection6] o.a.s.c.SolrCore [collection6]  CLOSING SolrCore org.apache.solr.core.SolrCore@4c0e1d8f
   [junit4]   2> 1007608 INFO  (coreCloseExecutor-3441-thread-7-processing-x:badConfig1) [    x:collection9] o.a.s.c.SolrCore [collection9]  CLOSING SolrCore org.apache.solr.core.SolrCore@2866a1f1
   [junit4]   2> 1007608 INFO  (coreCloseExecutor-3441-thread-6-processing-x:badConfig1) [    x:collection8] o.a.s.c.SolrCore [collection8]  CLOSING SolrCore org.apache.solr.core.SolrCore@614ac458
   [junit4]   2> 1007608 INFO  (coreCloseExecutor-3441-thread-7-processing-x:badConfig1) [    x:collection9] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection9
   [junit4]   2> 1007608 INFO  (coreCloseExecutor-3441-thread-4-processing-x:badConfig1) [    x:collection6] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection6
   [junit4]   2> 1007609 INFO  (coreCloseExecutor-3441-thread-6-processing-x:badConfig1) [    x:collection8] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection8
   [junit4]   2> 1007612 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[C9E5AF64279550E3]) [    x:badConfig1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1007614 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[C9E5AF64279550E3]) [    x:badConfig1] o.a.s.SolrTestCaseJ4 ###Ending testCachingLimit
   [junit4]   2> 1007615 INFO  (SUITE-TestLazyCores-seed#[C9E5AF64279550E3]-worker) [    x:badConfig1] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 1007615 INFO  (SUITE-TestLazyCores-seed#[C9E5AF64279550E3]-worker) [    x:badConfig1] o.a.s.c.CoreContainer Shutting down CoreContainer instance=206659173
   [junit4]   2> 1007615 INFO  (SUITE-TestLazyCores-seed#[C9E5AF64279550E3]-worker) [    x:badConfig1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_C9E5AF64279550E3-001
   [junit4]   2> Feb 01, 2017 12:25:11 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> Feb 01, 2017 12:25:31 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   [junit4]   2>    1) Thread[id=7106, name=searcherExecutor-3197-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Feb 01, 2017 12:25:31 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=7106, name=searcherExecutor-3197-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2> Feb 01, 2017 12:25:34 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=7106, name=searcherExecutor-3197-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=false): {}, locale=ar-AE, timezone=America/Cambridge_Bay
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=2,free=6188984,total=480247808
   [junit4]   2> NOTE: All tests run in this JVM: [TestInPlaceUpdatesDistrib, HttpSolrCallGetCoreTest, TestCopyFieldCollectionResource, TestInitQParser, MetricUtilsTest, TestPartialUpdateDeduplication, ZkNodePropsTest, TestSurroundQueryParser, StressHdfsTest, OverseerTaskQueueTest, TestClassicSimilarityFactory, BigEndianAscendingWordDeserializerTest, TestCloudRecovery, TestFieldCacheSanityChecker, DistributedQueryElevationComponentTest, SolrTestCaseJ4Test, PKIAuthenticationIntegrationTest, TestLockTree, RecoveryAfterSoftCommitTest, TestSizeLimitedDistributedMap, BasicAuthIntegrationTest, TestIntervalFaceting, FastVectorHighlighterTest, TestRandomRequestDistribution, AnalysisErrorHandlingTest, MissingSegmentRecoveryTest, CreateCollectionCleanupTest, HdfsRecoverLeaseTest, ClassificationUpdateProcessorTest, DeleteNodeTest, TestMacros, StandardRequestHandlerTest, TestComplexPhraseQParserPlugin, SystemInfoHandlerTest, TestLeaderElectionWithEmptyReplica, TestSolrDynamicMBean, GraphQueryTest, SimplePostToolTest, NumericFieldsTest, CoreSorterTest, RequiredFieldsTest, HdfsRestartWhileUpdatingTest, SimpleFacetsTest, TestRawResponseWriter, TestConfigSetsAPIExclusivity, TestRandomDVFaceting, PluginInfoTest, SpatialFilterTest, BJQParserTest, TestConfigsApi, TestReplicationHandler, TestSolrCoreParser, TestTrackingShardHandlerFactory, SpatialHeatmapFacetsTest, TestFieldSortValues, ZkStateReaderTest, TestXIncludeConfig, QueryElevationComponentTest, TestUpdate, CollectionTooManyReplicasTest, ExitableDirectoryReaderTest, SolrMetricManagerTest, SOLR749Test, TestStressReorder, CleanupOldIndexTest, TestFastLRUCache, TestSQLHandlerNonCloud, TestSerializedLuceneMatchVersion, TestPostingsSolrHighlighter, PingRequestHandlerTest, DataDrivenBlockJoinTest, TriLevelCompositeIdRoutingTest, TestGraphTermsQParserPlugin, XsltUpdateRequestHandlerTest, SolrXmlInZkTest, BasicDistributedZkTest, SearchHandlerTest, CoreMergeIndexesAdminHandlerTest, TestLazyCores]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLazyCores -Dtests.seed=C9E5AF64279550E3 -Dtests.slow=true -Dtests.locale=ar-AE -Dtests.timezone=America/Cambridge_Bay -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J0 | TestLazyCores (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   [junit4]    >    1) Thread[id=7106, name=searcherExecutor-3197-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C9E5AF64279550E3]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=7106, name=searcherExecutor-3197-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C9E5AF64279550E3]:0)
   [junit4] Completed [173/691 (2!)] on J0 in 29.96s, 9 tests, 3 errors <<< FAILURES!

[...truncated 64109 lines...]


[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1108 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1108/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
FAILED:  org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.testDelegationTokenCancelFail

Error Message:
expected:<200> but was:<404>

Stack Trace:
java.lang.AssertionError: expected:<200> but was:<404>
	at __randomizedtesting.SeedInfo.seed([51325F23EA17466B:398D6A093A8D5487]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.cancelDelegationToken(TestDelegationWithHadoopAuth.java:128)
	at org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.testDelegationTokenCancelFail(TestDelegationWithHadoopAuth.java:290)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 12772 lines...]
   [junit4] Suite: org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_51325F23EA17466B-001/init-core-data-001
   [junit4]   2> 3137600 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[51325F23EA17466B]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields
   [junit4]   2> 3137601 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[51325F23EA17466B]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 3137603 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[51325F23EA17466B]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_51325F23EA17466B-001/tempDir-001
   [junit4]   2> 3137603 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[51325F23EA17466B]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 3137603 INFO  (Thread-8797) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 3137603 INFO  (Thread-8797) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 3137703 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[51325F23EA17466B]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:50983
   [junit4]   2> 3137718 INFO  (jetty-launcher-5049-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 3137719 INFO  (jetty-launcher-5049-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6e85495e{/solr,null,AVAILABLE}
   [junit4]   2> 3137721 INFO  (jetty-launcher-5049-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 3137722 INFO  (jetty-launcher-5049-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3abef2b1{/solr,null,AVAILABLE}
   [junit4]   2> 3137722 INFO  (jetty-launcher-5049-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@287b7723{SSL,[ssl, http/1.1]}{127.0.0.1:33917}
   [junit4]   2> 3137723 INFO  (jetty-launcher-5049-thread-2) [    ] o.e.j.s.Server Started @3141937ms
   [junit4]   2> 3137723 INFO  (jetty-launcher-5049-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33917}
   [junit4]   2> 3137723 ERROR (jetty-launcher-5049-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3137723 INFO  (jetty-launcher-5049-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 3137723 INFO  (jetty-launcher-5049-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3137723 INFO  (jetty-launcher-5049-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3137723 INFO  (jetty-launcher-5049-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-30T06:42:39.932Z
   [junit4]   2> 3137732 INFO  (jetty-launcher-5049-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@396dc291{SSL,[ssl, http/1.1]}{127.0.0.1:64435}
   [junit4]   2> 3137733 INFO  (jetty-launcher-5049-thread-1) [    ] o.e.j.s.Server Started @3141947ms
   [junit4]   2> 3137733 INFO  (jetty-launcher-5049-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=64435}
   [junit4]   2> 3137733 ERROR (jetty-launcher-5049-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3137733 INFO  (jetty-launcher-5049-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 3137733 INFO  (jetty-launcher-5049-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3137734 INFO  (jetty-launcher-5049-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3137734 INFO  (jetty-launcher-5049-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-30T06:42:39.943Z
   [junit4]   2> 3137734 INFO  (jetty-launcher-5049-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3137740 INFO  (jetty-launcher-5049-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3137742 INFO  (jetty-launcher-5049-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50983/solr
   [junit4]   2> 3137750 INFO  (jetty-launcher-5049-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50983/solr
   [junit4]   2> 3137785 INFO  (jetty-launcher-5049-thread-1) [n:127.0.0.1:64435_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:64435_solr
   [junit4]   2> 3137786 INFO  (jetty-launcher-5049-thread-2) [n:127.0.0.1:33917_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33917_solr
   [junit4]   2> 3137786 INFO  (jetty-launcher-5049-thread-1) [n:127.0.0.1:64435_solr    ] o.a.s.c.Overseer Overseer (id=97370672975904774-127.0.0.1:64435_solr-n_0000000000) starting
   [junit4]   2> 3137796 INFO  (jetty-launcher-5049-thread-1) [n:127.0.0.1:64435_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64435_solr
   [junit4]   2> 3137802 INFO  (zkCallback-5056-thread-1-processing-n:127.0.0.1:33917_solr) [n:127.0.0.1:33917_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3137803 INFO  (zkCallback-5056-thread-1-processing-n:127.0.0.1:33917_solr) [n:127.0.0.1:33917_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3137817 INFO  (zkCallback-5060-thread-1-processing-n:127.0.0.1:64435_solr) [n:127.0.0.1:64435_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3137879 INFO  (jetty-launcher-5049-thread-1) [n:127.0.0.1:64435_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HadoopAuthPlugin
   [junit4]   2> 3137880 INFO  (jetty-launcher-5049-thread-1) [n:127.0.0.1:64435_solr    ] o.a.s.s.HadoopAuthPlugin Params: {delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, signer.secret.provider.zookeeper.path=/token, type=simple, zk-dt-secret-manager.enable=true, signer.secret.provider=zookeeper}
   [junit4]   2> 3137883 INFO  (jetty-launcher-5049-thread-1) [n:127.0.0.1:64435_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 3137907 INFO  (jetty-launcher-5049-thread-1-EventThread) [n:127.0.0.1:64435_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 3137922 INFO  (jetty-launcher-5049-thread-2) [n:127.0.0.1:33917_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HadoopAuthPlugin
   [junit4]   2> 3137922 INFO  (jetty-launcher-5049-thread-2) [n:127.0.0.1:33917_solr    ] o.a.s.s.HadoopAuthPlugin Params: {delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, signer.secret.provider.zookeeper.path=/token, type=simple, zk-dt-secret-manager.enable=true, signer.secret.provider=zookeeper}
   [junit4]   2> 3137922 INFO  (jetty-launcher-5049-thread-2) [n:127.0.0.1:33917_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 3137924 INFO  (jetty-launcher-5049-thread-2-EventThread) [n:127.0.0.1:33917_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 3137928 INFO  (jetty-launcher-5049-thread-1) [n:127.0.0.1:64435_solr    ] o.a.s.c.CoreContainer PKIAuthenticationPlugin is managing internode requests
   [junit4]   2> 3137931 INFO  (jetty-launcher-5049-thread-1) [n:127.0.0.1:64435_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_51325F23EA17466B-001/tempDir-001/node1/.
   [junit4]   2> 3137947 INFO  (jetty-launcher-5049-thread-2) [n:127.0.0.1:33917_solr    ] o.a.s.c.CoreContainer PKIAuthenticationPlugin is managing internode requests
   [junit4]   2> 3137950 INFO  (jetty-launcher-5049-thread-2) [n:127.0.0.1:33917_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_51325F23EA17466B-001/tempDir-001/node2/.
   [junit4]   2> 3138142 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancelFail-seed#[51325F23EA17466B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancelFail
   [junit4]   2> 3138234 ERROR (qtp1582363927-30612) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_1]
   [junit4]   2> 3138237 ERROR (qtp219259020-30602) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_1]
   [junit4]   2> 3138252 ERROR (qtp1582363927-30617) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_2]
   [junit4]   2> 3138254 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancelFail-seed#[51325F23EA17466B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancelFail
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDelegationWithHadoopAuth -Dtests.method=testDelegationTokenCancelFail -Dtests.seed=51325F23EA17466B -Dtests.slow=true -Dtests.locale=und -Dtests.timezone=Etc/GMT-1 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.14s J1 | TestDelegationWithHadoopAuth.testDelegationTokenCancelFail <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<200> but was:<404>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([51325F23EA17466B:398D6A093A8D5487]:0)
   [junit4]    > 	at org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.cancelDelegationToken(TestDelegationWithHadoopAuth.java:128)
   [junit4]    > 	at org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.testDelegationTokenCancelFail(TestDelegationWithHadoopAuth.java:290)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 3138294 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
   [junit4]   2> 3138317 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3138319 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50983/solr ready
   [junit4]   2> 3138325 INFO  (qtp1582363927-30609) [n:127.0.0.1:33917_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHj3igFaEij89wMCFP61qXM4hfu9zKx9ANIu7t3asoSMB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3138325 INFO  (qtp1582363927-30609) [n:127.0.0.1:33917_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHj3igFaEij89wMCFP61qXM4hfu9zKx9ANIu7t3asoSMB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3138337 INFO  (qtp1582363927-30616) [n:127.0.0.1:33917_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHj3igFaEij89wMCFP61qXM4hfu9zKx9ANIu7t3asoSMB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3138337 INFO  (qtp1582363927-30616) [n:127.0.0.1:33917_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHj3igFaEij89wMCFP61qXM4hfu9zKx9ANIu7t3asoSMB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3138342 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3138344 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50983/solr ready
   [junit4]   2> 3138368 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3138370 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50983/solr ready
   [junit4]   2> 3138373 INFO  (qtp1582363927-30609) [n:127.0.0.1:33917_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHkvigFaEij9LwQCFMlfwSUV0DtSFIQLy24HoJa2q5r5B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3138373 INFO  (qtp1582363927-30609) [n:127.0.0.1:33917_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHkvigFaEij9LwQCFMlfwSUV0DtSFIQLy24HoJa2q5r5B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3138387 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3138389 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50983/solr ready
   [junit4]   2> 3138393 INFO  (qtp1582363927-30617) [n:127.0.0.1:33917_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHkvigFaEij9LwQCFMlfwSUV0DtSFIQLy24HoJa2q5r5B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3138393 INFO  (qtp1582363927-30617) [n:127.0.0.1:33917_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHkvigFaEij9LwQCFMlfwSUV0DtSFIQLy24HoJa2q5r5B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3138398 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3138400 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50983/solr ready
   [junit4]   2> 3138431 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3138433 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50983/solr ready
   [junit4]   2> 3138436 INFO  (qtp1582363927-30617) [n:127.0.0.1:33917_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHlnigFaEij9ZwUCFG81eT9I2-gxf7fd1FI8M3STCKGeB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3138436 INFO  (qtp1582363927-30617) [n:127.0.0.1:33917_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHlnigFaEij9ZwUCFG81eT9I2-gxf7fd1FI8M3STCKGeB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3138445 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3138446 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50983/solr ready
   [junit4]   2> 3138460 INFO  (qtp1582363927-30616) [n:127.0.0.1:33917_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHlnigFaEij9ZwUCFG81eT9I2-gxf7fd1FI8M3STCKGeB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3138460 INFO  (qtp1582363927-30616) [n:127.0.0.1:33917_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHlnigFaEij9ZwUCFG81eT9I2-gxf7fd1FI8M3STCKGeB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3138487 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3138488 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50983/solr ready
   [junit4]   2> 3138491 INFO  (qtp1582363927-30617) [n:127.0.0.1:33917_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHmkigFaEij9pAYCFKRtqV8IuVKyB4RwLPywqNw8qyHrB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3138491 INFO  (qtp1582363927-30617) [n:127.0.0.1:33917_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHmkigFaEij9pAYCFKRtqV8IuVKyB4RwLPywqNw8qyHrB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3138497 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3138499 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50983/solr ready
   [junit4]   2> 3138507 INFO  (qtp1582363927-30618) [n:127.0.0.1:33917_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHmkigFaEij9pAYCFKRtqV8IuVKyB4RwLPywqNw8qyHrB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3138507 INFO  (qtp1582363927-30618) [n:127.0.0.1:33917_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHmkigFaEij9pAYCFKRtqV8IuVKyB4RwLPywqNw8qyHrB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3138513 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[51325F23EA17466B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
   [junit4]   2> 3138541 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[51325F23EA17466B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenVerify
   [junit4]   2> 3138550 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[51325F23EA17466B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3138559 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[51325F23EA17466B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50983/solr ready
   [junit4]   2> 3138581 INFO  (qtp1582363927-30608) [n:127.0.0.1:33917_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHnuigFaEij97gcCFMn8RGHgn_KSDjLAoMdx2pmpEYIVB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3138581 INFO  (qtp1582363927-30608) [n:127.0.0.1:33917_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHnuigFaEij97gcCFMn8RGHgn_KSDjLAoMdx2pmpEYIVB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3138590 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[51325F23EA17466B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3138592 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[51325F23EA17466B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50983/solr ready
   [junit4]   2> 3138603 INFO  (qtp1582363927-30617) [n:127.0.0.1:33917_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHnuigFaEij97gcCFMn8RGHgn_KSDjLAoMdx2pmpEYIVB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3138604 INFO  (qtp1582363927-30617) [n:127.0.0.1:33917_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHnuigFaEij97gcCFMn8RGHgn_KSDjLAoMdx2pmpEYIVB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3138608 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[51325F23EA17466B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3138609 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[51325F23EA17466B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50983/solr ready
   [junit4]   2> 3138614 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[51325F23EA17466B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenVerify
   [junit4]   2> 3138651 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[51325F23EA17466B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
   [junit4]   2> 3138659 WARN  (qtp219259020-30599) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: bar tries to renew a token with renewer foo
   [junit4]   2> 3138660 WARN  (qtp219259020-30599) [    ] o.a.s.s.HadoopAuthPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 3138661 WARN  (qtp1582363927-30605) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: bar tries to renew a token with renewer foo
   [junit4]   2> 3138661 WARN  (qtp1582363927-30605) [    ] o.a.s.s.HadoopAuthPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 3138674 WARN  (qtp219259020-30596) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 3138674 WARN  (qtp219259020-30596) [    ] o.a.s.s.HadoopAuthPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 3138675 WARN  (qtp1582363927-30608) [n:127.0.0.1:33917_solr    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 3138675 WARN  (qtp1582363927-30608) [n:127.0.0.1:33917_solr    ] o.a.s.s.HadoopAuthPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 3138675 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[51325F23EA17466B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenewFail
   [junit4]   2> 3138708 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenSolrClient-seed#[51325F23EA17466B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenSolrClient
   [junit4]   2> 3138725 INFO  (qtp219259020-30599) [n:127.0.0.1:64435_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHqWigFaEij-lgoCFHVV2KCjEwQxOm9-0FpreLvW6zXBB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3138725 INFO  (qtp219259020-30599) [n:127.0.0.1:64435_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVnuHHqWigFaEij-lgoCFHVV2KCjEwQxOm9-0FpreLvW6zXBB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3138726 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenSolrClient-seed#[51325F23EA17466B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenSolrClient
   [junit4]   2> 3138754 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[51325F23EA17466B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancel
   [junit4]   2> 3138767 ERROR (qtp219259020-30600) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_11]
   [junit4]   2> 3138791 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[51325F23EA17466B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3138792 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[51325F23EA17466B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50983/solr ready
   [junit4]   2> 3138800 ERROR (qtp1582363927-30605) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_12]
   [junit4]   2> 3138808 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[51325F23EA17466B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3138810 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[51325F23EA17466B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50983/solr ready
   [junit4]   2> 3138818 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[51325F23EA17466B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancel
   [junit4]   2> 3138845 INFO  (TEST-TestDelegationWithHadoopAuth.testZNodePaths-seed#[51325F23EA17466B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testZNodePaths
   [junit4]   2> 3138853 INFO  (TEST-TestDelegationWithHadoopAuth.testZNodePaths-seed#[51325F23EA17466B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testZNodePaths
   [junit4]   2> 3138879 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationOtherOp-seed#[51325F23EA17466B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationOtherOp
   [junit4]   2> 3138884 INFO  (qtp219259020-30599) [n:127.0.0.1:64435_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params op=someSolrOperation&action=LIST&user.name=foo&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3138884 INFO  (qtp219259020-30599) [n:127.0.0.1:64435_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={op=someSolrOperation&action=LIST&user.name=foo&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 3138885 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationOtherOp-seed#[51325F23EA17466B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationOtherOp
   [junit4]   2> 3138887 INFO  (jetty-launcher-5049-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@287b7723{SSL,[ssl, http/1.1]}{127.0.0.1:0}
   [junit4]   2> 3138888 INFO  (jetty-launcher-5049-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@396dc291{SSL,[ssl, http/1.1]}{127.0.0.1:0}
   [junit4]   2> 3138889 INFO  (jetty-launcher-5049-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1998659672
   [junit4]   2> 3138889 INFO  (jetty-launcher-5049-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1974556436
   [junit4]   2> 3138898 INFO  (jetty-launcher-5049-thread-2) [    ] o.a.s.c.Overseer Overseer (id=97370672975904774-127.0.0.1:64435_solr-n_0000000000) closing
   [junit4]   2> 3138899 INFO  (OverseerStateUpdate-97370672975904774-127.0.0.1:64435_solr-n_0000000000) [n:127.0.0.1:64435_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:64435_solr
   [junit4]   2> 3138901 INFO  (zkCallback-5056-thread-2-processing-n:127.0.0.1:33917_solr) [n:127.0.0.1:33917_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 3138902 INFO  (zkCallback-5056-thread-1-processing-n:127.0.0.1:33917_solr) [n:127.0.0.1:33917_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33917_solr
   [junit4]   2> 3138902 WARN  (zkCallback-5056-thread-2-processing-n:127.0.0.1:33917_solr) [n:127.0.0.1:33917_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 3140401 WARN  (zkCallback-5060-thread-3-processing-n:127.0.0.1:64435_solr) [n:127.0.0.1:64435_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 3140401 INFO  (jetty-launcher-5049-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 3140402 ERROR (Thread[Thread-8817,5,TGRP-TestDelegationWithHadoopAuth]) [n:127.0.0.1:64435_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 3140403 INFO  (jetty-launcher-5049-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 3140405 ERROR (Thread[Thread-8820,5,TGRP-TestDelegationWithHadoopAuth]) [n:127.0.0.1:33917_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 3140409 INFO  (jetty-launcher-5049-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6e85495e{/solr,null,UNAVAILABLE}
   [junit4]   2> 3140415 INFO  (jetty-launcher-5049-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3abef2b1{/solr,null,UNAVAILABLE}
   [junit4]   2> 3140418 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[51325F23EA17466B]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:50983 50983
   [junit4]   2> 3145446 INFO  (Thread-8797) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:50983 50983
   [junit4]   2> 3145447 WARN  (Thread-8797) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	16	/solr/aliases.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	16	/solr/clusterprops.json
   [junit4]   2> 	16	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_7
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_6
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_9
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_8
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_5
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_13
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_12
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_11
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_10
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	16	/solr/live_nodes
   [junit4]   2> 	16	/solr/collections
   [junit4]   2> 	4	/solr/overseer/queue
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 
   [junit4]   2> 3145447 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[51325F23EA17466B]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_51325F23EA17466B-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1626, maxMBSortInHeap=7.945100733465538, sim=RandomSimilarity(queryNorm=false): {}, locale=und, timezone=Etc/GMT-1
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=93911352,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [NoCacheHeaderTest, AutoCommitTest, BadIndexSchemaTest, TestTestInjection, ReplaceNodeTest, TestFiltering, JSONWriterTest, TestCustomSort, InfoHandlerTest, ShardRoutingCustomTest, TestInfoStreamLogging, TestExclusionRuleCollectionAccess, TestQueryUtils, HighlighterConfigTest, SolrGangliaReporterTest, ExitableDirectoryReaderTest, TestSolrDeletionPolicy2, ResponseHeaderTest, TestDocTermOrds, DistributedVersionInfoTest, StatsReloadRaceTest, ChaosMonkeySafeLeaderTest, FacetPivotSmallTest, TestSchemaResource, LeaderFailureAfterFreshStartTest, TestSolrCloudWithKerberosAlt, CoreAdminRequestStatusTest, DistributedTermsComponentTest, TestFuzzyAnalyzedSuggestions, SolrMetricReporterTest, TestStressCloudBlindAtomicUpdates, TestMissingGroups, TestHighFrequencyDictionaryFactory, UpdateRequestProcessorFactoryTest, TestGraphMLResponseWriter, TestQueryTypes, SpellCheckComponentTest, ConvertedLegacyTest, JavabinLoaderTest, ExternalFileFieldSortTest, FieldMutatingUpdateProcessorTest, TestNumericTerms64, TestLeaderInitiatedRecoveryThread, EnumFieldTest, QueryResultKeyTest, TestQuerySenderListener, TestCollationFieldDocValues, UUIDFieldTest, DistributedIntervalFacetingTest, DateMathParserTest, BlockJoinFacetDistribTest, TestDocSet, ConnectionReuseTest, TestZkChroot, RestartWhileUpdatingTest, TestTrie, TestStressLiveNodes, LeaderElectionIntegrationTest, BasicZkTest, TestReplicationHandlerBackup, SharedFSAutoReplicaFailoverUtilsTest, TestCloudDeleteByQuery, DistribCursorPagingTest, SSLMigrationTest, PreAnalyzedUpdateProcessorTest, TestUseDocValuesAsStored2, TestLegacyFieldCache, RegexBoostProcessorTest, SecurityConfHandlerTest, MergeStrategyTest, TestConfigOverlay, SolrCoreMetricManagerTest, DistributedQueryComponentOptimizationTest, TestRandomRequestDistribution, ReturnFieldsTest, SolrInfoMBeanTest, LukeRequestHandlerTest, RulesTest, TestAtomicUpdateErrorCases, TestConfig, FileBasedSpellCheckerTest, ZkSolrClientTest, NotRequiredUniqueKeyTest, TestDistributedMissingSort, MissingSegmentRecoveryTest, TestIndexingPerformance, TestOverriddenPrefixQueryForCustomFieldType, TestShardHandlerFactory, TestManagedResource, CoreAdminHandlerTest, TestRTimerTree, DistributedExpandComponentTest, AddBlockUpdateTest, QueryParsingTest, SolrCmdDistributorTest, TestStressRecovery, LeaderElectionContextKeyTest, CacheHeaderTest, OutOfBoxZkACLAndCredentialsProvidersTest, SolrJmxReporterTest, CollectionReloadTest, TestDistributedGrouping, TestFieldCacheVsDocValues, TestSolrCloudSnapshots, TestDFISimilarityFactory, TestSimpleTrackingShardHandler, MetricUtilsTest, TestMinMaxOnMultiValuedField, TestConfigSetProperties, IndexSchemaTest, SuggesterWFSTTest, TestFieldCacheSort, TestSolrQueryParserDefaultOperatorResource, RequestLoggingTest, BooleanFieldTest, TestSolrIndexConfig, TestReloadDeadlock, DeleteInactiveReplicaTest, CollectionsAPISolrJTest, TestHdfsUpdateLog, TestSQLHandlerNonCloud, URLClassifyProcessorTest, DateRangeFieldTest, TestComponentsName, HdfsUnloadDistributedZkTest, TestSuggestSpellingConverter, WrapperMergePolicyFactoryTest, FullHLLTest, SortSpecParsingTest, TestCloudPivotFacet, DeleteShardTest, DistribJoinFromCollectionTest, TestSegmentSorting, TestSolr4Spatial, TestRealTimeGet, TestSolrCloudWithHadoopAuthPlugin, RequestHandlersTest, TestFoldingMultitermQuery, CircularListTest, UpdateParamsTest, CreateCollectionCleanupTest, TestFieldResource, CSVRequestHandlerTest, MinimalSchemaTest, TestCoreDiscovery, TestReload, TestSolrFieldCacheMBean, SignatureUpdateProcessorFactoryTest, AliasIntegrationTest, BlockDirectoryTest, BitVectorTest, TestTolerantUpdateProcessorRandomCloud, TestJsonRequest, DistanceUnitsTest, PreAnalyzedFieldTest, SliceStateTest, ShardSplitTest, SpellPossibilityIteratorTest, BlockJoinFacetSimpleTest, BasicDistributedZk2Test, TestDFRSimilarityFactory, DirectUpdateHandlerOptimizeTest, LoggingHandlerTest, TestDistribDocBasedVersion, DistributedFacetPivotSmallAdvancedTest, CollectionsAPIDistributedZkTest, SyncSliceTest, OpenCloseCoreStressTest, OverseerTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, TestRandomFaceting, TestRandomDVFaceting, ZkCLITest, TestFaceting, TestRecovery, TermVectorComponentDistributedTest, TestJoin, TestRangeQuery, TestCoreContainer, PeerSyncTest, BasicFunctionalityTest, TestBadConfig, TestIndexSearcher, ShowFileRequestHandlerTest, CurrencyFieldOpenExchangeTest, SolrIndexSplitterTest, AnalysisAfterCoreReloadTest, SpellCheckCollatorTest, SuggesterTest, TestCSVLoader, SchemaVersionSpecificBehaviorTest, TestPseudoReturnFields, TestWordDelimiterFilterFactory, DefaultValueUpdateProcessorTest, SortByFunctionTest, DistanceFunctionTest, DebugComponentTest, TestWriterPerf, PrimitiveFieldTypeTest, TermsComponentTest, TestValueSourceCache, MoreLikeThisHandlerTest, TestSolrQueryParser, TestCollationField, TestAnalyzedSuggestions, BinaryUpdateRequestHandlerTest, CopyFieldTest, SolrIndexConfigTest, TestSearchPerf, TestPostingsSolrHighlighter, DirectSolrConnectionTest, TestPhraseSuggestions, TestCodecSupport, TestSweetSpotSimilarityFactory, TestLMDirichletSimilarityFactory, TestBM25SimilarityFactory, TestJmxMonitoredMap, PluginInfoTest, TestSystemIdResolver, PrimUtilsTest, SpellingQueryConverterTest, DOMUtilTest, RAMDirectoryFactoryTest, ClusterStateTest, TestLRUCache, TestUtils, TestDocumentBuilder, ZkNodePropsTest, SystemInfoHandlerTest, FileUtilsTest, DistributedMLTComponentTest, TestRTGBase, TestCrossCoreJoin, TestHighlightDedupGrouping, TestEmbeddedSolrServerConstructors, TestJettySolrRunner, ActionThrottleTest, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrBootstrapTest, CleanupOldIndexTest, ConfigSetsAPITest, CustomCollectionTest, DeleteStatusTest, HttpPartitionTest, LeaderInitiatedRecoveryOnShardRestartTest, MigrateRouteKeyTest, RecoveryAfterSoftCommitTest, SolrCloudExampleTest, TestCloudRecovery, TestConfigSetsAPI, TestConfigSetsAPIExclusivity, TestHdfsCloudBackupRestore, TestLeaderElectionZkExpiry, TestOnReconnectListenerSupport, TestReplicaProperties, TestSSLRandomization, TestShortCircuitedRequests, TestSolrCloudWithSecureImpersonation, TriLevelCompositeIdRoutingTest, HdfsBasicDistributedZk2Test, HdfsChaosMonkeyNothingIsSafeTest, HdfsChaosMonkeySafeLeaderTest, HdfsCollectionsAPIDistributedZkTest, HdfsRecoverLeaseTest, HdfsTlogReplayBufferedWhileIndexingTest, StressHdfsTest, TestImplicitCoreProperties, TestInitParams, TestNRTOpen, TestReloadAndDeleteDocs, TestSolrDynamicMBean, TestSolrCoreSnapshots, TestConfigReload, TestSolrConfigHandlerCloud, MetricsHandlerTest, SegmentsInfoRequestHandlerTest, DistributedDebugComponentTest, DistributedFacetPivotLargeTest, DistributedFacetPivotSmallTest, ReplicaListTransformerTest, ShufflingReplicaListTransformerTest, SuggestComponentContextFilterQueryTest, TestPivotHelperCode, TestUnifiedSolrHighlighter, TestSlowCompositeReaderWrapper, SolrMetricManagerTest, TestIntervalFaceting, TestMacroExpander, TestMacros, TestBinaryResponseWriter, TestExportWriter, TestJavabinTupleStreamParser, TestPushWriter, TestRawResponseWriter, TestRawTransformer, TestSolrQueryResponse, TestChildDocTransformer, TestSubQueryTransformer, TestSubQueryTransformerCrossCore, TestManagedResourceStorage, TestRestManager, TestBulkSchemaAPI, TestCopyFieldCollectionResource, TestDefaultSearchFieldResource, TestDynamicFieldResource, TestFieldCollectionResource, TestFieldTypeResource, TestSchemaNameResource, TestSchemaSimilarityResource, TestSchemaVersionResource, TestSerializedLuceneMatchVersion, TestUniqueKeyFieldResource, TestManagedSynonymFilterFactory, BadCopyFieldTest, DocValuesMissingTest, ManagedSchemaRoundRobinCloudTest, PreAnalyzedFieldManagedSchemaCloudTest, TestCloudSchemaless, TestReRankQParserPlugin, TestSearcherReuse, TestSimpleQParserPlugin, TestSolr4Spatial2, TestSolrCoreParser, TestStandardQParsers, TestStressUserVersions, TestTrieFacet, TestXmlQParser, TestXmlQParserPlugin, TestJsonFacetRefinement, TestJsonFacets, TestOrdValues, TestSortByMinMaxFunction, GraphQueryTest, TestScoreJoinQPNoScore, TestScoreJoinQPScore, CloudMLTQParserTest, SimpleMLTQParserTest, TestClassicSimilarityFactory, TestPerFieldSimilarityWithDefaultOverride, TestDefaultStatsCache, TestDistribIDF, TestExactSharedStatsCache, TestExactStatsCache, TestLRUStatsCache, BasicAuthStandaloneTest, PKIAuthenticationIntegrationTest, TestRuleBasedAuthorizationPlugin, TestSha256AuthenticationProvider, TestDelegationWithHadoopAuth]
   [junit4] Completed [649/684 (1!)] on J1 in 7.86s, 8 tests, 1 failure <<< FAILURES!

[...truncated 62578 lines...]


[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1107 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1107/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC

1 tests failed.
FAILED:  org.apache.solr.cloud.PeerSyncReplicationTest.test

Error Message:
timeout waiting to see all nodes active

Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
	at __randomizedtesting.SeedInfo.seed([97FCDF2BFE93A79B:1FA8E0F1506FCA63]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
	at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
	at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
	at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11157 lines...]
   [junit4] Suite: org.apache.solr.cloud.PeerSyncReplicationTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/init-core-data-001
   [junit4]   2> 612856 INFO  (SUITE-PeerSyncReplicationTest-seed#[97FCDF2BFE93A79B]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields
   [junit4]   2> 612858 INFO  (SUITE-PeerSyncReplicationTest-seed#[97FCDF2BFE93A79B]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 612858 INFO  (SUITE-PeerSyncReplicationTest-seed#[97FCDF2BFE93A79B]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 612862 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 612862 INFO  (Thread-1932) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 612862 INFO  (Thread-1932) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 612964 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.ZkTestServer start zk server on port:33242
   [junit4]   2> 612982 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 612985 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 612988 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 612990 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 612993 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 612995 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 612997 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 612999 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 613001 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 613004 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 613006 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 613194 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/control-001/cores/collection1
   [junit4]   2> 613196 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 613200 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5f764b72{/,null,AVAILABLE}
   [junit4]   2> 613202 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@78776131{HTTP/1.1,[http/1.1]}{127.0.0.1:36189}
   [junit4]   2> 613202 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.e.j.s.Server Started @616764ms
   [junit4]   2> 613202 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/tempDir-001/control/data, hostContext=/, hostPort=36189, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/control-001/cores}
   [junit4]   2> 613202 ERROR (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 613202 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 613203 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 613203 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 613203 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-29T19:25:24.124Z
   [junit4]   2> 613208 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 613209 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/control-001/solr.xml
   [junit4]   2> 613217 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33242/solr
   [junit4]   2> 613281 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [n:127.0.0.1:36189_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:36189_
   [junit4]   2> 613284 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [n:127.0.0.1:36189_    ] o.a.s.c.Overseer Overseer (id=97368009853173764-127.0.0.1:36189_-n_0000000000) starting
   [junit4]   2> 613301 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [n:127.0.0.1:36189_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36189_
   [junit4]   2> 613303 INFO  (zkCallback-627-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 613419 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [n:127.0.0.1:36189_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/control-001/cores
   [junit4]   2> 613419 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [n:127.0.0.1:36189_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 613423 INFO  (OverseerStateUpdate-97368009853173764-127.0.0.1:36189_-n_0000000000) [n:127.0.0.1:36189_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 613452 WARN  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 613453 INFO  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 613490 INFO  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 613575 WARN  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 613578 INFO  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 613596 INFO  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 613596 INFO  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/control-001/cores/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/control-001/cores/collection1/data/]
   [junit4]   2> 613596 INFO  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f0b559f
   [junit4]   2> 613598 INFO  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=23, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 613608 WARN  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 613621 INFO  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 613621 INFO  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 613623 INFO  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 613623 INFO  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 613623 INFO  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=35, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=15.822265625, floorSegmentMB=0.470703125, forceMergeDeletesPctAllowed=17.769356059393367, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2796911870719601
   [junit4]   2> 613624 INFO  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@79f551b4[collection1] main]
   [junit4]   2> 613626 INFO  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 613626 INFO  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 613626 INFO  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 613628 INFO  (searcherExecutor-2016-thread-1-processing-n:127.0.0.1:36189_ x:collection1 c:control_collection) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@79f551b4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 613628 INFO  (coreLoadExecutor-2015-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_ c:control_collection   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557888158451892224
   [junit4]   2> 613639 INFO  (coreZkRegister-2008-thread-1-processing-n:127.0.0.1:36189_ x:collection1 c:control_collection) [n:127.0.0.1:36189_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 613640 INFO  (coreZkRegister-2008-thread-1-processing-n:127.0.0.1:36189_ x:collection1 c:control_collection) [n:127.0.0.1:36189_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 613640 INFO  (coreZkRegister-2008-thread-1-processing-n:127.0.0.1:36189_ x:collection1 c:control_collection) [n:127.0.0.1:36189_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:36189/collection1/
   [junit4]   2> 613640 INFO  (coreZkRegister-2008-thread-1-processing-n:127.0.0.1:36189_ x:collection1 c:control_collection) [n:127.0.0.1:36189_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 613640 INFO  (coreZkRegister-2008-thread-1-processing-n:127.0.0.1:36189_ x:collection1 c:control_collection) [n:127.0.0.1:36189_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:36189/collection1/ has no replicas
   [junit4]   2> 613647 INFO  (coreZkRegister-2008-thread-1-processing-n:127.0.0.1:36189_ x:collection1 c:control_collection) [n:127.0.0.1:36189_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:36189/collection1/ shard1
   [junit4]   2> 613804 INFO  (coreZkRegister-2008-thread-1-processing-n:127.0.0.1:36189_ x:collection1 c:control_collection) [n:127.0.0.1:36189_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 613952 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 613954 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33242/solr ready
   [junit4]   2> 613954 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 614122 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-1-001/cores/collection1
   [junit4]   2> 614122 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-1-001
   [junit4]   2> 614123 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 614126 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5cc53fd9{/,null,AVAILABLE}
   [junit4]   2> 614126 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7083e0d4{HTTP/1.1,[http/1.1]}{127.0.0.1:46079}
   [junit4]   2> 614126 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.e.j.s.Server Started @617689ms
   [junit4]   2> 614127 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=46079, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-1-001/cores}
   [junit4]   2> 614127 ERROR (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 614129 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 614129 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 614129 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 614129 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-29T19:25:25.050Z
   [junit4]   2> 614134 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 614134 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-1-001/solr.xml
   [junit4]   2> 614144 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33242/solr
   [junit4]   2> 614162 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [n:127.0.0.1:46079_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 614171 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [n:127.0.0.1:46079_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46079_
   [junit4]   2> 614175 INFO  (zkCallback-631-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 614178 INFO  (zkCallback-627-thread-2-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 614180 INFO  (zkCallback-636-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 614290 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [n:127.0.0.1:46079_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-1-001/cores
   [junit4]   2> 614290 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [n:127.0.0.1:46079_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 614294 INFO  (OverseerStateUpdate-97368009853173764-127.0.0.1:36189_-n_0000000000) [n:127.0.0.1:36189_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 615317 WARN  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 615318 INFO  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 615337 INFO  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 615501 WARN  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 615502 INFO  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 615521 INFO  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 615524 INFO  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-1-001/cores/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-1-001/cores/collection1/data/]
   [junit4]   2> 615524 INFO  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f0b559f
   [junit4]   2> 615526 INFO  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=23, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 615534 WARN  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 615558 INFO  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 615558 INFO  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 615560 INFO  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 615560 INFO  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 615560 INFO  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=35, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=15.822265625, floorSegmentMB=0.470703125, forceMergeDeletesPctAllowed=17.769356059393367, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2796911870719601
   [junit4]   2> 615561 INFO  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@d913d0[collection1] main]
   [junit4]   2> 615562 INFO  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 615563 INFO  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 615563 INFO  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 615565 INFO  (searcherExecutor-2027-thread-1-processing-n:127.0.0.1:46079_ x:collection1 c:collection1) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@d913d0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 615565 INFO  (coreLoadExecutor-2026-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_ c:collection1   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557888160482983936
   [junit4]   2> 615573 INFO  (coreZkRegister-2021-thread-1-processing-n:127.0.0.1:46079_ x:collection1 c:collection1) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 615573 INFO  (coreZkRegister-2021-thread-1-processing-n:127.0.0.1:46079_ x:collection1 c:collection1) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 615574 INFO  (coreZkRegister-2021-thread-1-processing-n:127.0.0.1:46079_ x:collection1 c:collection1) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:46079/collection1/
   [junit4]   2> 615574 INFO  (coreZkRegister-2021-thread-1-processing-n:127.0.0.1:46079_ x:collection1 c:collection1) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 615574 INFO  (coreZkRegister-2021-thread-1-processing-n:127.0.0.1:46079_ x:collection1 c:collection1) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:46079/collection1/ has no replicas
   [junit4]   2> 615582 INFO  (coreZkRegister-2021-thread-1-processing-n:127.0.0.1:46079_ x:collection1 c:collection1) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:46079/collection1/ shard1
   [junit4]   2> 615734 INFO  (coreZkRegister-2021-thread-1-processing-n:127.0.0.1:46079_ x:collection1 c:collection1) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 615995 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-2-001/cores/collection1
   [junit4]   2> 615996 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-2-001
   [junit4]   2> 615997 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 615999 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@52d0d33a{/,null,AVAILABLE}
   [junit4]   2> 615999 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7b922797{HTTP/1.1,[http/1.1]}{127.0.0.1:32941}
   [junit4]   2> 615999 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.e.j.s.Server Started @619562ms
   [junit4]   2> 616000 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=32941, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-2-001/cores}
   [junit4]   2> 616000 ERROR (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 616001 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 616001 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 616002 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 616002 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-29T19:25:26.923Z
   [junit4]   2> 616007 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 616007 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-2-001/solr.xml
   [junit4]   2> 616015 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33242/solr
   [junit4]   2> 616051 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [n:127.0.0.1:32941_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 616067 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [n:127.0.0.1:32941_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32941_
   [junit4]   2> 616070 INFO  (zkCallback-636-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 616072 INFO  (zkCallback-627-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 616072 INFO  (zkCallback-631-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 616075 INFO  (zkCallback-642-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 616202 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [n:127.0.0.1:32941_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-2-001/cores
   [junit4]   2> 616202 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [n:127.0.0.1:32941_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 616205 INFO  (OverseerStateUpdate-97368009853173764-127.0.0.1:36189_-n_0000000000) [n:127.0.0.1:36189_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 617239 WARN  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 617242 INFO  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 617277 INFO  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 617364 WARN  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 617366 INFO  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 617388 INFO  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 617388 INFO  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-2-001/cores/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-2-001/cores/collection1/data/]
   [junit4]   2> 617388 INFO  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f0b559f
   [junit4]   2> 617391 INFO  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=23, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 617403 WARN  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 617419 INFO  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 617419 INFO  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 617421 INFO  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 617421 INFO  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 617422 INFO  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=35, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=15.822265625, floorSegmentMB=0.470703125, forceMergeDeletesPctAllowed=17.769356059393367, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2796911870719601
   [junit4]   2> 617423 INFO  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3dc4488[collection1] main]
   [junit4]   2> 617425 INFO  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 617425 INFO  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 617427 INFO  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 617429 INFO  (searcherExecutor-2038-thread-1-processing-n:127.0.0.1:32941_ x:collection1 c:collection1) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3dc4488[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 617429 INFO  (coreLoadExecutor-2037-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557888162437529600
   [junit4]   2> 617436 INFO  (coreZkRegister-2032-thread-1-processing-n:127.0.0.1:32941_ x:collection1 c:collection1) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
   [junit4]   2> 617436 INFO  (updateExecutor-639-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 617437 INFO  (recoveryExecutor-640-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 617437 INFO  (recoveryExecutor-640-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 617437 INFO  (recoveryExecutor-640-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
   [junit4]   2> 617437 INFO  (recoveryExecutor-640-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 617437 INFO  (recoveryExecutor-640-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:46079/collection1/] and I am [http://127.0.0.1:32941/collection1/]
   [junit4]   2> 617441 INFO  (recoveryExecutor-640-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:46079]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:32941_&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 617448 INFO  (qtp459992681-5160) [n:127.0.0.1:46079_    ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
   [junit4]   2> 617449 INFO  (qtp459992681-5160) [n:127.0.0.1:46079_    ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
   [junit4]   2> 617449 INFO  (qtp459992681-5160) [n:127.0.0.1:46079_    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:32941_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:32941","node_name":"127.0.0.1:32941_","state":"down"}
   [junit4]   2> 617885 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-3-001/cores/collection1
   [junit4]   2> 617886 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-3-001
   [junit4]   2> 617887 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 617888 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@45d04d9{/,null,AVAILABLE}
   [junit4]   2> 617889 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@19326ef2{HTTP/1.1,[http/1.1]}{127.0.0.1:38462}
   [junit4]   2> 617889 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.e.j.s.Server Started @621452ms
   [junit4]   2> 617889 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=38462, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-3-001/cores}
   [junit4]   2> 617889 ERROR (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 617890 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 617890 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 617890 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 617890 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-29T19:25:28.811Z
   [junit4]   2> 617896 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 617896 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-3-001/solr.xml
   [junit4]   2> 617904 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33242/solr
   [junit4]   2> 617925 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [n:127.0.0.1:38462_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 617935 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [n:127.0.0.1:38462_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38462_
   [junit4]   2> 617938 INFO  (zkCallback-627-thread-1-processing-n:127.0.0.1:36189_) [n:127.0.0.1:36189_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 617938 INFO  (zkCallback-636-thread-1-processing-n:127.0.0.1:46079_) [n:127.0.0.1:46079_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 617941 INFO  (zkCallback-631-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 617941 INFO  (zkCallback-642-thread-1-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 617943 INFO  (zkCallback-649-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 618031 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [n:127.0.0.1:38462_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-3-001/cores
   [junit4]   2> 618031 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [n:127.0.0.1:38462_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 618034 INFO  (OverseerStateUpdate-97368009853173764-127.0.0.1:36189_-n_0000000000) [n:127.0.0.1:36189_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 618449 INFO  (qtp459992681-5160) [n:127.0.0.1:46079_    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:32941_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:32941","node_name":"127.0.0.1:32941_","state":"recovering"}
   [junit4]   2> 618449 INFO  (qtp459992681-5160) [n:127.0.0.1:46079_    ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
   [junit4]   2> 618449 INFO  (qtp459992681-5160) [n:127.0.0.1:46079_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:32941_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1001
   [junit4]   2> 619047 WARN  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 619048 INFO  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 619084 INFO  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 619157 WARN  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 619159 INFO  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 619177 INFO  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 619177 INFO  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-3-001/cores/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001/shard-3-001/cores/collection1/data/]
   [junit4]   2> 619177 INFO  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f0b559f
   [junit4]   2> 619179 INFO  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=23, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 619187 WARN  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 619201 INFO  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 619201 INFO  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 619203 INFO  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 619203 INFO  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 619204 INFO  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=35, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=15.822265625, floorSegmentMB=0.470703125, forceMergeDeletesPctAllowed=17.769356059393367, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2796911870719601
   [junit4]   2> 619204 INFO  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@489c5b2f[collection1] main]
   [junit4]   2> 619206 INFO  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 619207 INFO  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 619207 INFO  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 619208 INFO  (searcherExecutor-2049-thread-1-processing-n:127.0.0.1:38462_ x:collection1 c:collection1) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@489c5b2f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 619209 INFO  (coreLoadExecutor-2048-thread-1-processing-n:127.0.0.1:38462_) [n:127.0.0.1:38462_ c:collection1   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557888164303994880
   [junit4]   2> 619214 INFO  (coreZkRegister-2043-thread-1-processing-n:127.0.0.1:38462_ x:collection1 c:collection1) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
   [junit4]   2> 619215 INFO  (updateExecutor-646-thread-1-processing-n:127.0.0.1:38462_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 619215 INFO  (recoveryExecutor-647-thread-1-processing-n:127.0.0.1:38462_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 619216 INFO  (recoveryExecutor-647-thread-1-processing-n:127.0.0.1:38462_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 619216 INFO  (recoveryExecutor-647-thread-1-processing-n:127.0.0.1:38462_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
   [junit4]   2> 619216 INFO  (recoveryExecutor-647-thread-1-processing-n:127.0.0.1:38462_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 619216 INFO  (recoveryExecutor-647-thread-1-processing-n:127.0.0.1:38462_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:46079/collection1/] and I am [http://127.0.0.1:38462/collection1/]
   [junit4]   2> 619220 INFO  (recoveryExecutor-647-thread-1-processing-n:127.0.0.1:38462_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:46079]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:38462_&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 619222 INFO  (qtp459992681-5165) [n:127.0.0.1:46079_    ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
   [junit4]   2> 619223 INFO  (qtp459992681-5165) [n:127.0.0.1:46079_    ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
   [junit4]   2> 619223 INFO  (qtp459992681-5165) [n:127.0.0.1:46079_    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:38462_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:38462","node_name":"127.0.0.1:38462_","state":"down"}
   [junit4]   2> 619553 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 619554 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 30 for each attempt
   [junit4]   2> 619554 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
   [junit4]   2> 620225 INFO  (qtp459992681-5165) [n:127.0.0.1:46079_    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:38462_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:38462","node_name":"127.0.0.1:38462_","state":"recovering"}
   [junit4]   2> 620225 INFO  (qtp459992681-5165) [n:127.0.0.1:46079_    ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
   [junit4]   2> 620225 INFO  (qtp459992681-5165) [n:127.0.0.1:46079_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:38462_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1003
   [junit4]   2> 625450 INFO  (recoveryExecutor-640-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:46079/collection1/] - recoveringAfterStartup=[true]
   [junit4]   2> 625450 INFO  (recoveryExecutor-640-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:32941 START replicas=[http://127.0.0.1:46079/collection1/] nUpdates=1000
   [junit4]   2> 625454 INFO  (qtp459992681-5158) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
   [junit4]   2> 625454 INFO  (qtp459992681-5158) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1]  webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 625456 INFO  (recoveryExecutor-640-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
   [junit4]   2> 625456 INFO  (recoveryExecutor-640-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync 
   [junit4]   2> 625456 INFO  (recoveryExecutor-640-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 625456 INFO  (recoveryExecutor-640-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 625458 INFO  (recoveryExecutor-640-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 625458 INFO  (recoveryExecutor-640-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
   [junit4]   2> 625458 INFO  (recoveryExecutor-640-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
   [junit4]   2> 625458 INFO  (recoveryExecutor-640-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
   [junit4]   2> 625458 INFO  (recoveryExecutor-640-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
   [junit4]   2> 627226 INFO  (recoveryExecutor-647-thread-1-processing-n:127.0.0.1:38462_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:46079/collection1/] - recoveringAfterStartup=[true]
   [junit4]   2> 627226 INFO  (recoveryExecutor-647-thread-1-processing-n:127.0.0.1:38462_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:38462 START replicas=[http://127.0.0.1:46079/collection1/] nUpdates=1000
   [junit4]   2> 627228 INFO  (qtp459992681-5164) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
   [junit4]   2> 627229 INFO  (qtp459992681-5164) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1]  webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 627230 INFO  (recoveryExecutor-647-thread-1-processing-n:127.0.0.1:38462_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
   [junit4]   2> 627230 INFO  (recoveryExecutor-647-thread-1-processing-n:127.0.0.1:38462_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync 
   [junit4]   2> 627230 INFO  (recoveryExecutor-647-thread-1-processing-n:127.0.0.1:38462_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 627230 INFO  (recoveryExecutor-647-thread-1-processing-n:127.0.0.1:38462_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 627231 INFO  (recoveryExecutor-647-thread-1-processing-n:127.0.0.1:38462_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 627231 INFO  (recoveryExecutor-647-thread-1-processing-n:127.0.0.1:38462_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
   [junit4]   2> 627231 INFO  (recoveryExecutor-647-thread-1-processing-n:127.0.0.1:38462_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
   [junit4]   2> 627231 INFO  (recoveryExecutor-647-thread-1-processing-n:127.0.0.1:38462_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
   [junit4]   2> 627231 INFO  (recoveryExecutor-647-thread-1-processing-n:127.0.0.1:38462_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
   [junit4]   2> 627574 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
   [junit4]   2> 627578 INFO  (qtp404506833-5122) [n:127.0.0.1:36189_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 627578 INFO  (qtp404506833-5122) [n:127.0.0.1:36189_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 627579 INFO  (qtp404506833-5122) [n:127.0.0.1:36189_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 627580 INFO  (qtp404506833-5122) [n:127.0.0.1:36189_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 1
   [junit4]   2> 627584 INFO  (qtp459992681-5165) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 627584 INFO  (qtp459992681-5165) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 627585 INFO  (qtp459992681-5165) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 627585 INFO  (qtp459992681-5165) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:46079/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
   [junit4]   2> 627585 INFO  (qtp1452201829-5221) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 627587 INFO  (qtp2039228070-5186) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 627588 INFO  (qtp1452201829-5221) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 627589 INFO  (qtp2039228070-5186) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 627589 INFO  (qtp1452201829-5221) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 627589 INFO  (qtp1452201829-5221) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:46079/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 3
   [junit4]   2> 627589 INFO  (qtp2039228070-5186) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 627590 INFO  (qtp2039228070-5186) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:46079/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 3
   [junit4]   2> 627590 INFO  (qtp459992681-5161) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 9
   [junit4]   2> 627592 INFO  (qtp459992681-5160) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1]  webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 627594 INFO  (qtp2039228070-5190) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.S.Request [collection1]  webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 627596 INFO  (qtp1452201829-5224) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.S.Request [collection1]  webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 629601 INFO  (qtp404506833-5124) [n:127.0.0.1:36189_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1557888175197650944)} 0 3
   [junit4]   2> 629610 INFO  (qtp1452201829-5225) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1557888175201845248&distrib.from=http://127.0.0.1:46079/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557888175201845248)} 0 4
   [junit4]   2> 629612 INFO  (qtp2039228070-5191) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1557888175201845248&distrib.from=http://127.0.0.1:46079/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557888175201845248)} 0 6
   [junit4]   2> 629613 INFO  (qtp459992681-5158) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1557888175201845248)} 0 10
   [junit4]   2> 629621 INFO  (qtp2039228070-5193) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:46079/collection1/&wt=javabin&version=2}{add=[0 (1557888175217573888)]} 0 1
   [junit4]   2> 629622 INFO  (qtp1452201829-5226) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:46079/collection1/&wt=javabin&version=2}{add=[0 (1557888175217573888)]} 0 1
   [junit4]   2> 629622 INFO  (qtp459992681-5163) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[0 (1557888175217573888)]} 0 6
   [junit4]   2> 629626 INFO  (qtp2039228070-5192) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:46079/collection1/&wt=javabin&version=2}{add=[1 (1557888175224913920)]} 0 1
   [junit4]   2> 629627 INFO  (qtp1452201829-5227) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:46079/collection1/&wt=javabin&version=2}{add=[1 (1557888175224913920)]} 0 0
   [junit4]   2> 629627 INFO  (qtp459992681-5164) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[1 (1557888175224913920)]} 0 3
   [junit4]   2> 629629 INFO  (qtp1452201829-5227) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:46079/collection1/&wt=javabin&version=2}{add=[2 (1557888175230156800)]} 0 0
   [junit4]   2> 629630 INFO  (qtp2039228070-5192) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:46079/collection1/&wt=javabin&version=2}{add=[2 (1557888175230156800)]} 0 0
   [junit4]   2> 629631 INFO  (qtp459992681-5165) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[2 (1557888175230156800)]} 0 2
   [junit4]   2> 629632 INFO  (qtp1452201829-5227) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:46079/collection1/&wt=javabin&version=2}{add=[3 (1557888175232253952)]} 0 0
   [junit4]   2> 629633 INFO  (qtp2039228070-5186) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:46079/collection1/&wt=javabin&version=2}{add=[3 (1557888175232253952)]} 0 0
   [junit4]   2> 629633 INFO  (qtp459992681-5165) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[3 (1557888175232253952)]} 0 2
   [junit4]   2> 629635 INFO  (qtp1452201829-5227) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:46079/collection1/&wt=javabin&version=2}{add=[4 (1557888175235399680)]} 0 0
   [junit4]   2> 629636 INFO  (qtp2039228070-5189) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:46079/collection1/&wt=javabin&version=2}{add=[4 (1557888175235399680)]} 0 0
   [junit4]   2> 629636 INFO  (qtp459992681-5162) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[4 (1557888175235399680)]} 0 2
   [junit4]   2> 629638 INFO  (qtp2039228070-5190) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:46079/collection1/&wt=javabin&version=2}{add=[5 (1557888175238545408)]} 0 0
   [junit4]   2> 629638 INFO  (qtp1452201829-5227) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:46079/collection1/&wt=javabin&version=2}{add=[5 (1557888175238545408)]} 0 0
   [junit4]   2> 629638 INFO  (qtp459992681-5160) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[5 (1557888175238545408)]} 0 1
   [junit4]   2> 629640 INFO  (qtp1452201829-5227) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:46079/collection1/&wt=javabin&version=2}{add=[6 (1557888175240642560)]} 0 0
   [junit4]   2> 629640 INFO  (qtp2039228070-5190) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:46079/collection1/&wt=javabin&version=2}{add=[6 (1557888175240642560)]} 0 0
   [junit4]   2> 629640 INFO  (qtp459992681-5158) [n:127.0.0.1:46079_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={wt=javabin&version=2}{add=[6 (1557888175240642560)]} 0 1
   [junit4]   2> 629642 INFO  (qtp1452201829-5227) [n:127.0.0.1:38462_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:46079/collection1/&wt=javabin&version=2}{add=[7 (1557888175242739712)]} 0 0
   [junit4]   2> 629642 INFO  (qtp2039228070-5193) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp= path=/update params={update.distrib=FROMLEADER&dis

[...truncated too long message...]

eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 813728 INFO  (qtp459992681-5163) [n:127.0.0.1:46079_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:32941_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=400 QTime=181395
   [junit4]   2> 813728 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.ChaosMonkey monkey: stop shard! 32941
   [junit4]   2> 813729 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1240827987
   [junit4]   2> 813730 WARN  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.RecoveryStrategy Stopping recovery for core=[collection1] coreNodeName=[core_node2]
   [junit4]   2> 813736 WARN  (updateExecutor-653-thread-2-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Skipping recovery because Solr is shutdown
   [junit4]   2> 818075 INFO  (recoveryExecutor-654-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy RecoveryStrategy has been closed
   [junit4]   2> 818075 INFO  (recoveryExecutor-654-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Finished recovery process, successful=[false]
   [junit4]   2> 818075 INFO  (recoveryExecutor-654-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@550e4d8c
   [junit4]   2> 818076 WARN  (recoveryExecutor-654-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Stopping recovery for core=[collection1] coreNodeName=[core_node2]
   [junit4]   2> 818105 INFO  (recoveryExecutor-654-thread-1-processing-n:127.0.0.1:32941_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:32941_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 818107 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.Overseer Overseer (id=97368009853173777-127.0.0.1:32941_-n_0000000004) closing
   [junit4]   2> 818107 INFO  (OverseerStateUpdate-97368009853173777-127.0.0.1:32941_-n_0000000004) [n:127.0.0.1:32941_    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:32941_
   [junit4]   2> 818112 WARN  (zkCallback-656-thread-5-processing-n:127.0.0.1:32941_) [n:127.0.0.1:32941_    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 818113 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 818116 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@209342d7{HTTP/1.1,[http/1.1]}{127.0.0.1:32941}
   [junit4]   2> 818116 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@44cb8250{/,null,UNAVAILABLE}
   [junit4]   2> 818118 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.ChaosMonkey monkey: stop shard! 38462
   [junit4]   2> 818122 INFO  (TEST-PeerSyncReplicationTest.test-seed#[97FCDF2BFE93A79B]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:33242 33242
   [junit4]   2> 818155 INFO  (Thread-1932) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:33242 33242
   [junit4]   2> 818157 WARN  (Thread-1932) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/aliases.json
   [junit4]   2> 	6	/solr/clusterprops.json
   [junit4]   2> 	5	/solr/security.json
   [junit4]   2> 	5	/solr/configs/conf1
   [junit4]   2> 	4	/solr/collections/collection1/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/collections/collection1/leader_elect/shard1/election/97368009853173768-core_node1-n_0000000000
   [junit4]   2> 	2	/solr/overseer_elect/election/97368009853173768-127.0.0.1:46079_-n_0000000001
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	205	/solr/overseer/collection-queue-work
   [junit4]   2> 	42	/solr/overseer/queue
   [junit4]   2> 	6	/solr/collections
   [junit4]   2> 	5	/solr/live_nodes
   [junit4]   2> 	4	/solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=PeerSyncReplicationTest -Dtests.method=test -Dtests.seed=97FCDF2BFE93A79B -Dtests.slow=true -Dtests.locale=ga -Dtests.timezone=Africa/Lagos -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE  205s J0 | PeerSyncReplicationTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: timeout waiting to see all nodes active
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([97FCDF2BFE93A79B:1FA8E0F1506FCA63]:0)
   [junit4]    > 	at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
   [junit4]    > 	at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
   [junit4]    > 	at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
   [junit4]    > 	at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 818163 INFO  (SUITE-PeerSyncReplicationTest-seed#[97FCDF2BFE93A79B]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_97FCDF2BFE93A79B-001
   [junit4]   2> Jan 29, 2017 7:28:49 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=29297, maxDocsPerChunk=493, blockSize=3), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=29297, blockSize=3)), sim=RandomSimilarity(queryNorm=true): {}, locale=ga, timezone=Africa/Lagos
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=376897728,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [SpatialHeatmapFacetsTest, TestBlendedInfixSuggestions, SimpleFacetsTest, TestMacros, TlogReplayBufferedWhileIndexingTest, ChaosMonkeyNothingIsSafeTest, RecoveryZkTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestFastLRUCache, TestJettySolrRunner, SimplePostToolTest, TestQueryWrapperFilter, CurrencyFieldXmlFileTest, TestFreeTextSuggestions, BJQParserTest, SOLR749Test, DateFieldTest, TestPKIAuthenticationPlugin, LeaderInitiatedRecoveryOnCommitTest, TestInPlaceUpdatesDistrib, AnalysisErrorHandlingTest, DistributedDebugComponentTest, MultiThreadedOCPTest, ConcurrentDeleteAndCreateCollectionTest, SynonymTokenizerTest, DistributedQueryComponentCustomSortTest, TestUpdate, DocumentAnalysisRequestHandlerTest, XsltUpdateRequestHandlerTest, TestSchemaNameResource, TestElisionMultitermQuery, HighlighterMaxOffsetTest, JvmMetricsTest, CdcrReplicationDistributedZkTest, UpdateLogTest, TestStressInPlaceUpdates, BlockCacheTest, TestGroupingSearch, TestIBSimilarityFactory, SolrTestCaseJ4Test, TestFaceting, TestNamedUpdateProcessors, TestMiniSolrCloudCluster, TestCSVLoader, TestXIncludeConfig, DocValuesTest, SolrMetricManagerTest, DistributedQueryElevationComponentTest, SolrGraphiteReporterTest, CopyFieldTest, HdfsRecoveryZkTest, TestQuerySenderNoQuery, PeerSyncReplicationTest]
   [junit4] Completed [130/684 (1!)] on J0 in 206.35s, 1 test, 1 failure <<< FAILURES!

[...truncated 64170 lines...]


[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1106 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1106/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC

4 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPISolrJTest

Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:48438 within 30000 ms

Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:48438 within 30000 ms
	at __randomizedtesting.SeedInfo.seed([1B683D8CD4394BB0]:0)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:182)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:116)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:111)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:98)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForAllNodes(MiniSolrCloudCluster.java:260)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:254)
	at org.apache.solr.cloud.SolrCloudTestCase$Builder.configure(SolrCloudTestCase.java:188)
	at org.apache.solr.cloud.CollectionsAPISolrJTest.setupCluster(CollectionsAPISolrJTest.java:50)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:847)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:48438 within 30000 ms
	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:233)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:174)
	... 31 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPISolrJTest

Error Message:
47 threads leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPISolrJTest:     1) Thread[id=8920, name=qtp981723713-8920, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=8930, name=qtp879764203-8930, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=8923, name=qtp116894854-8923-acceptor-0@39e36bab-ServerConnector@32ab0058{HTTP/1.1,[http/1.1]}{127.0.0.1:61337}, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=8910, name=jetty-launcher-1356-thread-2, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=8918, name=qtp981723713-8918, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=8932, name=qtp879764203-8932, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=8901, name=Thread-2988, state=WAITING, group=TGRP-CollectionsAPISolrJTest]         at java.lang.Object.wait(Native Method)         at java.lang.Thread.join(Thread.java:1249)         at java.lang.Thread.join(Thread.java:1323)         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)    8) Thread[id=8929, name=qtp879764203-8929, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=8936, name=qtp590191702-8936, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=8950, name=org.eclipse.jetty.server.session.HashSessionManager@2c489694Timer, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   11) Thread[id=8931, name=qtp879764203-8931, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   12) Thread[id=8903, name=SessionTracker, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at java.lang.Object.wait(Native Method)         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)   13) Thread[id=8942, name=qtp590191702-8942, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   14) Thread[id=8960, name=Scheduler-876872545, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   15) Thread[id=8949, name=qtp116894854-8949, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   16) Thread[id=8927, name=qtp879764203-8927, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:745)   17) Thread[id=8916, name=qtp981723713-8916, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   18) Thread[id=8944, name=qtp590191702-8944, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   19) Thread[id=8947, name=qtp116894854-8947, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   20) Thread[id=8909, name=jetty-launcher-1356-thread-1, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   21) Thread[id=8948, name=qtp116894854-8948, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   22) Thread[id=8935, name=qtp590191702-8935, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   23) Thread[id=8943, name=org.eclipse.jetty.server.session.HashSessionManager@785e692fTimer, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   24) Thread[id=8915, name=qtp981723713-8915-acceptor-0@48d98632-ServerConnector@63428659{HTTP/1.1,[http/1.1]}{127.0.0.1:48438}, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:745)   25) Thread[id=8928, name=qtp879764203-8928-acceptor-0@1b8de8df-ServerConnector@31ceaa7b{HTTP/1.1,[http/1.1]}{127.0.0.1:35207}, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:745)   26) Thread[id=8938, name=qtp590191702-8938, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   27) Thread[id=8941, name=qtp590191702-8941, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   28) Thread[id=8902, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)         at java.lang.Thread.run(Thread.java:745)   29) Thread[id=8937, name=qtp590191702-8937-acceptor-0@1782bf77-ServerConnector@581bdcb1{HTTP/1.1,[http/1.1]}{127.0.0.1:41205}, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:745)   30) Thread[id=8912, name=jetty-launcher-1356-thread-4, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   31) Thread[id=8914, name=qtp981723713-8914, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   32) Thread[id=8922, name=qtp116894854-8922, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:745)   33) Thread[id=8925, name=qtp879764203-8925, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   34) Thread[id=8921, name=org.eclipse.jetty.server.session.HashSessionManager@300a3c9dTimer, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   35) Thread[id=8946, name=qtp116894854-8946, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   36) Thread[id=8934, name=org.eclipse.jetty.server.session.HashSessionManager@7f12a507Timer, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   37) Thread[id=8917, name=qtp981723713-8917, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   38) Thread[id=8945, name=qtp590191702-8945, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   39) Thread[id=8953, name=qtp590191702-8953, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   40) Thread[id=8926, name=qtp116894854-8926, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   41) Thread[id=8911, name=jetty-launcher-1356-thread-3, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   42) Thread[id=8933, name=qtp879764203-8933, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   43) Thread[id=8905, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)   44) Thread[id=8919, name=qtp981723713-8919, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:745)   45) Thread[id=8913, name=qtp981723713-8913, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   46) Thread[id=8924, name=qtp116894854-8924, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)   47) Thread[id=8940, name=qtp590191702-8940, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 47 threads leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPISolrJTest: 
   1) Thread[id=8920, name=qtp981723713-8920, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=8930, name=qtp879764203-8930, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=8923, name=qtp116894854-8923-acceptor-0@39e36bab-ServerConnector@32ab0058{HTTP/1.1,[http/1.1]}{127.0.0.1:61337}, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=8910, name=jetty-launcher-1356-thread-2, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=8918, name=qtp981723713-8918, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=8932, name=qtp879764203-8932, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=8901, name=Thread-2988, state=WAITING, group=TGRP-CollectionsAPISolrJTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Thread.join(Thread.java:1249)
        at java.lang.Thread.join(Thread.java:1323)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
        at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
        at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   8) Thread[id=8929, name=qtp879764203-8929, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=8936, name=qtp590191702-8936, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=8950, name=org.eclipse.jetty.server.session.HashSessionManager@2c489694Timer, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  11) Thread[id=8931, name=qtp879764203-8931, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  12) Thread[id=8903, name=SessionTracker, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at java.lang.Object.wait(Native Method)
        at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
  13) Thread[id=8942, name=qtp590191702-8942, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  14) Thread[id=8960, name=Scheduler-876872545, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  15) Thread[id=8949, name=qtp116894854-8949, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  16) Thread[id=8927, name=qtp879764203-8927, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:745)
  17) Thread[id=8916, name=qtp981723713-8916, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  18) Thread[id=8944, name=qtp590191702-8944, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  19) Thread[id=8947, name=qtp116894854-8947, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  20) Thread[id=8909, name=jetty-launcher-1356-thread-1, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  21) Thread[id=8948, name=qtp116894854-8948, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  22) Thread[id=8935, name=qtp590191702-8935, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  23) Thread[id=8943, name=org.eclipse.jetty.server.session.HashSessionManager@785e692fTimer, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  24) Thread[id=8915, name=qtp981723713-8915-acceptor-0@48d98632-ServerConnector@63428659{HTTP/1.1,[http/1.1]}{127.0.0.1:48438}, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:745)
  25) Thread[id=8928, name=qtp879764203-8928-acceptor-0@1b8de8df-ServerConnector@31ceaa7b{HTTP/1.1,[http/1.1]}{127.0.0.1:35207}, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:745)
  26) Thread[id=8938, name=qtp590191702-8938, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  27) Thread[id=8941, name=qtp590191702-8941, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  28) Thread[id=8902, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
        at java.lang.Thread.run(Thread.java:745)
  29) Thread[id=8937, name=qtp590191702-8937-acceptor-0@1782bf77-ServerConnector@581bdcb1{HTTP/1.1,[http/1.1]}{127.0.0.1:41205}, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:745)
  30) Thread[id=8912, name=jetty-launcher-1356-thread-4, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  31) Thread[id=8914, name=qtp981723713-8914, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  32) Thread[id=8922, name=qtp116894854-8922, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:745)
  33) Thread[id=8925, name=qtp879764203-8925, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  34) Thread[id=8921, name=org.eclipse.jetty.server.session.HashSessionManager@300a3c9dTimer, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  35) Thread[id=8946, name=qtp116894854-8946, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  36) Thread[id=8934, name=org.eclipse.jetty.server.session.HashSessionManager@7f12a507Timer, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  37) Thread[id=8917, name=qtp981723713-8917, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  38) Thread[id=8945, name=qtp590191702-8945, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  39) Thread[id=8953, name=qtp590191702-8953, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  40) Thread[id=8926, name=qtp116894854-8926, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  41) Thread[id=8911, name=jetty-launcher-1356-thread-3, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  42) Thread[id=8933, name=qtp879764203-8933, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  43) Thread[id=8905, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)
  44) Thread[id=8919, name=qtp981723713-8919, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:745)
  45) Thread[id=8913, name=qtp981723713-8913, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  46) Thread[id=8924, name=qtp116894854-8924, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
  47) Thread[id=8940, name=qtp590191702-8940, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([1B683D8CD4394BB0]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPISolrJTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=8943, name=org.eclipse.jetty.server.session.HashSessionManager@785e692fTimer, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=8960, name=Scheduler-876872545, state=WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=8902, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=8927, name=qtp879764203-8927, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]         at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)         at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)         at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=8910, name=jetty-launcher-1356-thread-2, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=8911, name=jetty-launcher-1356-thread-3, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=8912, name=jetty-launcher-1356-thread-4, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=8909, name=jetty-launcher-1356-thread-1, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.ut

[...truncated too long message...]

  8) Thread[id=8909, name=jetty-launcher-1356-thread-1, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    9) Thread[id=8922, name=qtp116894854-8922, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]
   [junit4]    >         at sun.nio.ch.IOUtil.drain(Native Method)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:109)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   10) Thread[id=8921, name=org.eclipse.jetty.server.session.HashSessionManager@300a3c9dTimer, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   11) Thread[id=8919, name=qtp981723713-8919, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   12) Thread[id=8936, name=qtp590191702-8936, state=RUNNABLE, group=TGRP-CollectionsAPISolrJTest]
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   13) Thread[id=8950, name=org.eclipse.jetty.server.session.HashSessionManager@2c489694Timer, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   14) Thread[id=8934, name=org.eclipse.jetty.server.session.HashSessionManager@7f12a507Timer, state=TIMED_WAITING, group=TGRP-CollectionsAPISolrJTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([1B683D8CD4394BB0]:0)
   [junit4] Completed [173/684 (2!)] on J1 in 86.44s, 0 tests, 3 errors <<< FAILURES!

[...truncated 64052 lines...]


[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1105 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1105/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

2 tests failed.
FAILED:  org.apache.solr.handler.extraction.TestExtractionDateUtil.testParseDate

Error Message:
Incorrect parsed timestamp: 1226583351000 != 1226579751000 (Thu Nov 13 04:35:51 AKST 2008)

Stack Trace:
java.lang.AssertionError: Incorrect parsed timestamp: 1226583351000 != 1226579751000 (Thu Nov 13 04:35:51 AKST 2008)
	at __randomizedtesting.SeedInfo.seed([72FED8BB5D4CC3BE:38E7A08E26E5B40B]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.handler.extraction.TestExtractionDateUtil.assertParsedDate(TestExtractionDateUtil.java:59)
	at org.apache.solr.handler.extraction.TestExtractionDateUtil.testParseDate(TestExtractionDateUtil.java:54)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail

Error Message:
expected:<200> but was:<404>

Stack Trace:
java.lang.AssertionError: expected:<200> but was:<404>
	at __randomizedtesting.SeedInfo.seed([7FB8FEC24945F57B:1707CBE899DFE797]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:140)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:304)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11268 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_7FB8FEC24945F57B-001/init-core-data-001
   [junit4]   2> 1034991 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[7FB8FEC24945F57B]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields
   [junit4]   2> 1034993 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[7FB8FEC24945F57B]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1034995 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[7FB8FEC24945F57B]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_7FB8FEC24945F57B-001/tempDir-001
   [junit4]   2> 1034996 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[7FB8FEC24945F57B]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1034996 INFO  (Thread-3712) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1034996 INFO  (Thread-3712) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1035096 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[7FB8FEC24945F57B]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:35032
   [junit4]   2> 1035104 INFO  (jetty-launcher-2467-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1035105 INFO  (jetty-launcher-2467-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1035105 INFO  (jetty-launcher-2467-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@11b8a4b3{/solr,null,AVAILABLE}
   [junit4]   2> 1035107 INFO  (jetty-launcher-2467-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@45bf5f97{/solr,null,AVAILABLE}
   [junit4]   2> 1035112 INFO  (jetty-launcher-2467-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6e367cef{HTTP/1.1,[http/1.1]}{127.0.0.1:39119}
   [junit4]   2> 1035112 INFO  (jetty-launcher-2467-thread-1) [    ] o.e.j.s.Server Started @1038542ms
   [junit4]   2> 1035112 INFO  (jetty-launcher-2467-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39119}
   [junit4]   2> 1035112 ERROR (jetty-launcher-2467-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1035112 INFO  (jetty-launcher-2467-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 1035113 INFO  (jetty-launcher-2467-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1035113 INFO  (jetty-launcher-2467-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1035113 INFO  (jetty-launcher-2467-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-28T23:54:37.230Z
   [junit4]   2> 1035113 INFO  (jetty-launcher-2467-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@228a333{HTTP/1.1,[http/1.1]}{127.0.0.1:35651}
   [junit4]   2> 1035113 INFO  (jetty-launcher-2467-thread-2) [    ] o.e.j.s.Server Started @1038544ms
   [junit4]   2> 1035113 INFO  (jetty-launcher-2467-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35651}
   [junit4]   2> 1035114 ERROR (jetty-launcher-2467-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1035114 INFO  (jetty-launcher-2467-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 1035114 INFO  (jetty-launcher-2467-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1035114 INFO  (jetty-launcher-2467-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1035114 INFO  (jetty-launcher-2467-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-28T23:54:37.231Z
   [junit4]   2> 1035116 INFO  (jetty-launcher-2467-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1035117 INFO  (jetty-launcher-2467-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1035129 INFO  (jetty-launcher-2467-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35032/solr
   [junit4]   2> 1035132 INFO  (jetty-launcher-2467-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35032/solr
   [junit4]   2> 1035171 INFO  (jetty-launcher-2467-thread-1) [n:127.0.0.1:39119_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:39119_solr
   [junit4]   2> 1035173 INFO  (jetty-launcher-2467-thread-1) [n:127.0.0.1:39119_solr    ] o.a.s.c.Overseer Overseer (id=97363406167343110-127.0.0.1:39119_solr-n_0000000000) starting
   [junit4]   2> 1035182 INFO  (jetty-launcher-2467-thread-1) [n:127.0.0.1:39119_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39119_solr
   [junit4]   2> 1035186 INFO  (OverseerStateUpdate-97363406167343110-127.0.0.1:39119_solr-n_0000000000) [n:127.0.0.1:39119_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1035199 INFO  (zkCallback-2477-thread-1-processing-n:127.0.0.1:35651_solr) [n:127.0.0.1:35651_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1035202 INFO  (jetty-launcher-2467-thread-2) [n:127.0.0.1:35651_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35651_solr
   [junit4]   2> 1035204 INFO  (zkCallback-2477-thread-1-processing-n:127.0.0.1:35651_solr) [n:127.0.0.1:35651_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1035290 INFO  (zkCallback-2478-thread-1-processing-n:127.0.0.1:39119_solr) [n:127.0.0.1:39119_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1035403 INFO  (jetty-launcher-2467-thread-1) [n:127.0.0.1:39119_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 1035415 INFO  (jetty-launcher-2467-thread-1) [n:127.0.0.1:39119_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
   [junit4]   2> 1035486 INFO  (jetty-launcher-2467-thread-2) [n:127.0.0.1:35651_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 1035486 INFO  (jetty-launcher-2467-thread-2) [n:127.0.0.1:35651_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
   [junit4]   2> 1035574 INFO  (jetty-launcher-2467-thread-2) [n:127.0.0.1:35651_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 1035610 INFO  (jetty-launcher-2467-thread-1) [n:127.0.0.1:39119_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 1035701 INFO  (jetty-launcher-2467-thread-1-EventThread) [n:127.0.0.1:39119_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 1035706 INFO  (jetty-launcher-2467-thread-2-EventThread) [n:127.0.0.1:35651_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 1035925 INFO  (jetty-launcher-2467-thread-1) [n:127.0.0.1:39119_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_7FB8FEC24945F57B-001/tempDir-001/node1/.
   [junit4]   2> 1035938 INFO  (jetty-launcher-2467-thread-2) [n:127.0.0.1:35651_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_7FB8FEC24945F57B-001/tempDir-001/node2/.
   [junit4]   2> 1036569 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[7FB8FEC24945F57B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancelFail
   [junit4]   2> 1036654 ERROR (qtp1403892907-12271) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_1]
   [junit4]   2> 1036663 ERROR (qtp490647314-12265) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_1]
   [junit4]   2> 1036697 ERROR (qtp1403892907-12273) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_2]
   [junit4]   2> 1036699 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[7FB8FEC24945F57B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancelFail
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithDelegationTokens -Dtests.method=testDelegationTokenCancelFail -Dtests.seed=7FB8FEC24945F57B -Dtests.slow=true -Dtests.locale=tr-TR -Dtests.timezone=Israel -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.17s J0 | TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<200> but was:<404>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7FB8FEC24945F57B:1707CBE899DFE797]:0)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:140)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:304)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1036725 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[7FB8FEC24945F57B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenVerify
   [junit4]   2> 1036747 INFO  (qtp490647314-12259) [n:127.0.0.1:39119_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI16igFaC40RegMDFPaEWPrNnADjioVjy7I4lZurRMbiB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1036747 INFO  (qtp490647314-12259) [n:127.0.0.1:39119_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI16igFaC40RegMDFPaEWPrNnADjioVjy7I4lZurRMbiB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1036759 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1036763 INFO  (qtp1403892907-12269) [n:127.0.0.1:35651_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI16igFaC40RegMDFPaEWPrNnADjioVjy7I4lZurRMbiB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1036763 INFO  (qtp1403892907-12269) [n:127.0.0.1:35651_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI16igFaC40RegMDFPaEWPrNnADjioVjy7I4lZurRMbiB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1036773 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1036774 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35032/solr ready
   [junit4]   2> 1036777 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1036778 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[7FB8FEC24945F57B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenVerify
   [junit4]   2> 1036800 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[7FB8FEC24945F57B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancel
   [junit4]   2> 1036850 ERROR (qtp490647314-12259) [n:127.0.0.1:39119_solr    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_4]
   [junit4]   2> 1036861 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1036865 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1036867 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35032/solr ready
   [junit4]   2> 1036870 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1036889 ERROR (qtp1403892907-12272) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_5]
   [junit4]   2> 1036895 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1036897 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35032/solr ready
   [junit4]   2> 1036898 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1036903 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1036904 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35032/solr ready
   [junit4]   2> 1036907 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1036908 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[7FB8FEC24945F57B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancel
   [junit4]   2> 1036929 INFO  (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[7FB8FEC24945F57B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testZNodePaths
   [junit4]   2> 1036936 INFO  (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[7FB8FEC24945F57B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testZNodePaths
   [junit4]   2> 1036963 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[7FB8FEC24945F57B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenSolrClient
   [junit4]   2> 1036970 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1036973 INFO  (qtp490647314-12265) [n:127.0.0.1:39119_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI5pigFaC40SaQcDFAw8BpR60xcUiz3F8wi55SG-LINaB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1036973 INFO  (qtp490647314-12265) [n:127.0.0.1:39119_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI5pigFaC40SaQcDFAw8BpR60xcUiz3F8wi55SG-LINaB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1036984 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[7FB8FEC24945F57B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenSolrClient
   [junit4]   2> 1037005 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[7FB8FEC24945F57B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationOtherOp
   [junit4]   2> 1037010 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1037011 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35032/solr ready
   [junit4]   2> 1037013 INFO  (qtp1403892907-12268) [n:127.0.0.1:35651_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params op=someSolrOperation&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1037013 INFO  (qtp1403892907-12268) [n:127.0.0.1:35651_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={op=someSolrOperation&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1037013 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bar&p=bar&t=test&e=1485683679129&s=YqfFbPRQWxthG3+XTTTHZkMQre8="; Path=/; Domain=127.0.0.1; Expires=Paz, 29-Oca-2017 09:54:39 GMT; HttpOnly". Invalid 'expires' attribute: Paz, 29-Oca-2017 09:54:39 GMT
   [junit4]   2> 1037015 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[7FB8FEC24945F57B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationOtherOp
   [junit4]   2> 1037035 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
   [junit4]   2> 1037049 INFO  (qtp490647314-12258) [n:127.0.0.1:39119_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI6xigFaC40SsQgDFDusuaU0tmM7bI4cD5H-4dqcG3XsB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1037049 INFO  (qtp490647314-12258) [n:127.0.0.1:39119_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI6xigFaC40SsQgDFDusuaU0tmM7bI4cD5H-4dqcG3XsB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1037068 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1037071 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35032/solr ready
   [junit4]   2> 1037072 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1037075 INFO  (qtp1403892907-12268) [n:127.0.0.1:35651_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI6xigFaC40SsQgDFDusuaU0tmM7bI4cD5H-4dqcG3XsB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1037075 INFO  (qtp1403892907-12268) [n:127.0.0.1:35651_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI6xigFaC40SsQgDFDusuaU0tmM7bI4cD5H-4dqcG3XsB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1037080 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1037081 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35032/solr ready
   [junit4]   2> 1037083 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1037102 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1037103 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35032/solr ready
   [junit4]   2> 1037105 INFO  (qtp1403892907-12268) [n:127.0.0.1:35651_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI7higFaC40S4QkDFBH_OvUdNcVisZJUa1VDjWyLPKZWB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1037105 INFO  (qtp1403892907-12268) [n:127.0.0.1:35651_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI7higFaC40S4QkDFBH_OvUdNcVisZJUa1VDjWyLPKZWB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1037110 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1037112 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35032/solr ready
   [junit4]   2> 1037114 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1037118 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1037120 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35032/solr ready
   [junit4]   2> 1037121 INFO  (qtp1403892907-12269) [n:127.0.0.1:35651_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI7higFaC40S4QkDFBH_OvUdNcVisZJUa1VDjWyLPKZWB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1037121 INFO  (qtp1403892907-12269) [n:127.0.0.1:35651_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI7higFaC40S4QkDFBH_OvUdNcVisZJUa1VDjWyLPKZWB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1037124 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1037134 INFO  (qtp490647314-12258) [n:127.0.0.1:39119_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI8KigFaC40TCgoDFL7l2esuhr0wIdlQiDOFajf9WwxEB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1037134 INFO  (qtp490647314-12258) [n:127.0.0.1:39119_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI8KigFaC40TCgoDFL7l2esuhr0wIdlQiDOFajf9WwxEB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1037140 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1037147 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35032/solr ready
   [junit4]   2> 1037150 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1037168 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1037170 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35032/solr ready
   [junit4]   2> 1037171 INFO  (qtp1403892907-12267) [n:127.0.0.1:35651_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI8KigFaC40TCgoDFL7l2esuhr0wIdlQiDOFajf9WwxEB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1037171 INFO  (qtp1403892907-12267) [n:127.0.0.1:35651_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI8KigFaC40TCgoDFL7l2esuhr0wIdlQiDOFajf9WwxEB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1037247 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1037286 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35032/solr ready
   [junit4]   2> 1037295 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1037350 INFO  (qtp490647314-12265) [n:127.0.0.1:39119_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI_FigFaC40TxQsDFA7G347dnB0YxYAB__mT_AglYHr9B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1037350 INFO  (qtp490647314-12265) [n:127.0.0.1:39119_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI_FigFaC40TxQsDFA7G347dnB0YxYAB__mT_AglYHr9B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1037358 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1037361 INFO  (qtp1403892907-12271) [n:127.0.0.1:35651_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI_FigFaC40TxQsDFA7G347dnB0YxYAB__mT_AglYHr9B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1037361 INFO  (qtp1403892907-12271) [n:127.0.0.1:35651_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnngI_FigFaC40TxQsDFA7G347dnB0YxYAB__mT_AglYHr9B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1037374 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1037374 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[7FB8FEC24945F57B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
   [junit4]   2> 1037421 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[7FB8FEC24945F57B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
   [junit4]   2> 1037435 WARN  (qtp490647314-12264) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
   [junit4]   2> 1037436 WARN  (qtp490647314-12264) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 1037439 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1037444 WARN  (qtp1403892907-12272) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
   [junit4]   2> 1037444 WARN  (qtp1403892907-12272) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 1037445 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1037454 WARN  (qtp490647314-12263) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 1037455 WARN  (qtp490647314-12263) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 1037455 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1037456 WARN  (qtp1403892907-12274) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 1037456 WARN  (qtp1403892907-12274) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 1037456 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[7FB8FEC24945F57B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Per, 01-Oca-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Per, 01-Oca-1970 00:00:00 GMT
   [junit4]   2> 1037456 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[7FB8FEC24945F57B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenewFail
   [junit4]   2> 1037458 INFO  (jetty-launcher-2467-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@6e367cef{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1037462 INFO  (jetty-launcher-2467-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=291954587
   [junit4]   2> 1037478 INFO  (jetty-launcher-2467-thread-2) [    ] o.a.s.c.Overseer Overseer (id=97363406167343110-127.0.0.1:39119_solr-n_0000000000) closing
   [junit4]   2> 1037478 INFO  (OverseerStateUpdate-97363406167343110-127.0.0.1:39119_solr-n_0000000000) [n:127.0.0.1:39119_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:39119_solr
   [junit4]   2> 1037481 INFO  (jetty-launcher-2467-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@228a333{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1037493 INFO  (jetty-launcher-2467-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1192935250
   [junit4]   2> 1037495 INFO  (zkCallback-2477-thread-1-processing-n:127.0.0.1:35651_solr) [n:127.0.0.1:35651_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 1037496 INFO  (zkCallback-2477-thread-2-processing-n:127.0.0.1:35651_solr) [n:127.0.0.1:35651_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:35651_solr
   [junit4]   2> 1037498 INFO  (zkCallback-2477-thread-2-processing-n:127.0.0.1:35651_solr) [n:127.0.0.1:35651_solr    ] o.a.s.c.Overseer Overseer (id=97363406167343109-127.0.0.1:35651_solr-n_0000000001) starting
   [junit4]   2> 1037507 INFO  (jetty-launcher-2467-thread-1) [    ] o.a.s.c.Overseer Overseer (id=97363406167343109-127.0.0.1:35651_solr-n_0000000001) closing
   [junit4]   2> 1037508 INFO  (OverseerStateUpdate-97363406167343109-127.0.0.1:35651_solr-n_0000000001) [n:127.0.0.1:35651_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:35651_solr
   [junit4]   2> 1037528 INFO  (OverseerCollectionConfigSetProcessor-97363406167343109-127.0.0.1:35651_solr-n_0000000001) [n:127.0.0.1:35651_solr    ] o.a.s.c.OverseerTaskProcessor According to ZK I (id=97363406167343109-127.0.0.1:35651_solr-n_0000000001) am no longer a leader.
   [junit4]   2> 1037528 ERROR (OverseerCollectionConfigSetProcessor-97363406167343109-127.0.0.1:35651_solr-n_0000000001) [n:127.0.0.1:35651_solr    ] o.a.s.c.OverseerTaskProcessor :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer/collection-queue-work
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:339)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:336)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:336)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:299)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerTaskQueue.getTailId(OverseerTaskQueue.java:264)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:162)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 1039001 WARN  (zkCallback-2478-thread-2-processing-n:127.0.0.1:39119_solr) [n:127.0.0.1:39119_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1039001 INFO  (jetty-launcher-2467-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1039002 ERROR (Thread[Thread-3729,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:39119_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 1039010 WARN  (zkCallback-2477-thread-2-processing-n:127.0.0.1:35651_solr) [n:127.0.0.1:35651_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1039010 INFO  (jetty-launcher-2467-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@11b8a4b3{/solr,null,UNAVAILABLE}
   [junit4]   2> 1039010 INFO  (jetty-launcher-2467-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1039010 ERROR (Thread[Thread-3731,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:35651_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 1039016 INFO  (jetty-launcher-2467-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@45bf5f97{/solr,null,UNAVAILABLE}
   [junit4]   2> 1039019 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[7FB8FEC24945F57B]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35032 35032
   [junit4]   2> 1039047 INFO  (Thread-3712) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35032 35032
   [junit4]   2> 1039048 WARN  (Thread-3712) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	15	/solr/aliases.json
   [junit4]   2> 	15	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	15	/solr/clusterstate.json
   [junit4]   2> 	3	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_7
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_6
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_9
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_8
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_5
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_13
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_12
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_11
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_10
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	15	/solr/live_nodes
   [junit4]   2> 	15	/solr/collections
   [junit4]   2> 	3	/solr/overseer/queue
   [junit4]   2> 	3	/solr/overseer/collection-queue-work
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 
   [junit4]   2> 1039049 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[7FB8FEC24945F57B]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_7FB8FEC24945F57B-001
   [junit4]   2> Jan 28, 2017 11:54:41 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 3 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1960, maxMBSortInHeap=5.366507458569629, sim=RandomSimilarity(queryNorm=true): {}, locale=tr-TR, timezone=Israel
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=230094016,total=348823552
   [junit4]   2> NOTE: All tests run in this JVM: [TestClusterStateMutator, TestUninvertingReader, BaseCdcrDistributedZkTest, TestBlendedInfixSuggestions, TestGraphTermsQParserPlugin, TestFaceting, ExplicitHLLTest, IndexBasedSpellCheckerTest, TestDownShardTolerantSearch, SolrCloudExampleTest, BigEndianAscendingWordSerializerTest, DocumentBuilderTest, TestReloadAndDeleteDocs, TestLRUCache, TestDynamicFieldCollectionResource, TestExtendedDismaxParser, SegmentsInfoRequestHandlerTest, TestOrdValues, HighlighterMaxOffsetTest, TestSubQueryTransformerDistrib, TestAddFieldRealTimeGet, TestSchemaManager, DistributedFacetPivotLongTailTest, DistributedQueryComponentCustomSortTest, PathHierarchyTokenizerFactoryTest, TestFieldCache, SuggestComponentTest, LeaderFailoverAfterPartitionTest, OverriddenZkACLAndCredentialsProvidersTest, XsltUpdateRequestHandlerTest, TestExceedMaxTermLength, SyncSliceTest, BadCopyFieldTest, AssignTest, TestJsonFacets, DistributedQueryElevationComponentTest, CloudMLTQParserTest, TestMacros, AnalysisErrorHandlingTest, DistributedFacetPivotSmallTest, TestSubQueryTransformer, TestBlobHandler, TestCollationField, TestSolrConfigHandlerCloud, TestRTGBase, TestLeaderElectionZkExpiry, SOLR749Test, DistributedDebugComponentTest, TestCharFilters, RecoveryZkTest, CurrencyFieldXmlFileTest, LeaderInitiatedRecoveryOnCommitTest, TestNonDefinedSimilarityFactory, UpdateLogTest, TestRemoteStreaming, SolrTestCaseJ4Test, MultiTermTest, TestSSLRandomization, CheckHdfsIndexTest, TestCollapseQParserPlugin, OverseerModifyCollectionTest, PrimUtilsTest, TestIBSimilarityFactory, ZkCLITest, ActionThrottleTest, TestJettySolrRunner, TestFieldCacheSanityChecker, BasicDistributedZkTest, TestBulkSchemaAPI, TestCSVLoader, BlockJoinFacetRandomTest, BasicAuthIntegrationTest, TestShortCircuitedRequests, TestHashPartitioner, TestPostingsSolrHighlighter, TestFunctionQuery, TestPerFieldSimilarity, TestJmxIntegration, TestHdfsCloudBackupRestore, ClusterStateUpdateTest, TestStressReorder, OutputWriterTest, CloudExitableDirectoryReaderTest, TestDocumentBuilder, TestSQLHandler, TestSolrCloudWithDelegationTokens]
   [junit4] Completed [185/684 (1!)] on J0 in 5.09s, 8 tests, 1 failure <<< FAILURES!

[...truncated 6788 lines...]
   [junit4] Suite: org.apache.solr.handler.extraction.TestExtractionDateUtil
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestExtractionDateUtil -Dtests.method=testParseDate -Dtests.seed=72FED8BB5D4CC3BE -Dtests.slow=true -Dtests.locale=de-GR -Dtests.timezone=America/Metlakatla -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.12s J1 | TestExtractionDateUtil.testParseDate <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Incorrect parsed timestamp: 1226583351000 != 1226579751000 (Thu Nov 13 04:35:51 AKST 2008)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([72FED8BB5D4CC3BE:38E7A08E26E5B40B]:0)
   [junit4]    > 	at org.apache.solr.handler.extraction.TestExtractionDateUtil.assertParsedDate(TestExtractionDateUtil.java:59)
   [junit4]    > 	at org.apache.solr.handler.extraction.TestExtractionDateUtil.testParseDate(TestExtractionDateUtil.java:54)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=false): {}, locale=de-GR, timezone=America/Metlakatla
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=82155720,total=97320960
   [junit4]   2> NOTE: All tests run in this JVM: [TestExtractionDateUtil]
   [junit4] Completed [1/4 (1!)] on J1 in 1.24s, 2 tests, 1 failure <<< FAILURES!

[...truncated 57266 lines...]


[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1104 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1104/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail

Error Message:
expected:<200> but was:<404>

Stack Trace:
java.lang.AssertionError: expected:<200> but was:<404>
	at __randomizedtesting.SeedInfo.seed([6BC0DC5DB76DAA3B:37FE97767F7B8D7]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:140)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:294)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 12335 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_6BC0DC5DB76DAA3B-001/init-core-data-001
   [junit4]   2> 2563369 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[6BC0DC5DB76DAA3B]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields
   [junit4]   2> 2563370 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[6BC0DC5DB76DAA3B]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2563375 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[6BC0DC5DB76DAA3B]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_6BC0DC5DB76DAA3B-001/tempDir-001
   [junit4]   2> 2563375 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[6BC0DC5DB76DAA3B]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2563375 INFO  (Thread-4340) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2563375 INFO  (Thread-4340) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2563475 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[6BC0DC5DB76DAA3B]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:32927
   [junit4]   2> 2563485 INFO  (jetty-launcher-3829-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2563485 INFO  (jetty-launcher-3829-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2563488 INFO  (jetty-launcher-3829-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@499ae4e6{/solr,null,AVAILABLE}
   [junit4]   2> 2563488 INFO  (jetty-launcher-3829-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7d1b27b7{/solr,null,AVAILABLE}
   [junit4]   2> 2563490 INFO  (jetty-launcher-3829-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@131871a3{HTTP/1.1,[http/1.1]}{127.0.0.1:38940}
   [junit4]   2> 2563490 INFO  (jetty-launcher-3829-thread-1) [    ] o.e.j.s.Server Started @2566944ms
   [junit4]   2> 2563491 INFO  (jetty-launcher-3829-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38940}
   [junit4]   2> 2563490 INFO  (jetty-launcher-3829-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7612f839{HTTP/1.1,[http/1.1]}{127.0.0.1:51488}
   [junit4]   2> 2563491 ERROR (jetty-launcher-3829-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2563491 INFO  (jetty-launcher-3829-thread-2) [    ] o.e.j.s.Server Started @2566945ms
   [junit4]   2> 2563491 INFO  (jetty-launcher-3829-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 2563491 INFO  (jetty-launcher-3829-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=51488}
   [junit4]   2> 2563491 INFO  (jetty-launcher-3829-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2563491 INFO  (jetty-launcher-3829-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2563491 INFO  (jetty-launcher-3829-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-28T15:37:38.459Z
   [junit4]   2> 2563492 ERROR (jetty-launcher-3829-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2563492 INFO  (jetty-launcher-3829-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 2563492 INFO  (jetty-launcher-3829-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2563492 INFO  (jetty-launcher-3829-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2563492 INFO  (jetty-launcher-3829-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-28T15:37:38.460Z
   [junit4]   2> 2563496 INFO  (jetty-launcher-3829-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2563497 INFO  (jetty-launcher-3829-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2563506 INFO  (jetty-launcher-3829-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:32927/solr
   [junit4]   2> 2563507 INFO  (jetty-launcher-3829-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:32927/solr
   [junit4]   2> 2563551 INFO  (jetty-launcher-3829-thread-2) [n:127.0.0.1:51488_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:51488_solr
   [junit4]   2> 2563552 INFO  (jetty-launcher-3829-thread-1) [n:127.0.0.1:38940_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38940_solr
   [junit4]   2> 2563552 INFO  (jetty-launcher-3829-thread-2) [n:127.0.0.1:51488_solr    ] o.a.s.c.Overseer Overseer (id=97361451964497926-127.0.0.1:51488_solr-n_0000000000) starting
   [junit4]   2> 2563556 INFO  (zkCallback-3839-thread-1-processing-n:127.0.0.1:38940_solr) [n:127.0.0.1:38940_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2563558 INFO  (zkCallback-3840-thread-1-processing-n:127.0.0.1:51488_solr) [n:127.0.0.1:51488_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2563566 INFO  (jetty-launcher-3829-thread-2) [n:127.0.0.1:51488_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51488_solr
   [junit4]   2> 2563600 INFO  (zkCallback-3839-thread-1-processing-n:127.0.0.1:38940_solr) [n:127.0.0.1:38940_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2563601 INFO  (zkCallback-3840-thread-1-processing-n:127.0.0.1:51488_solr) [n:127.0.0.1:51488_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2563660 INFO  (jetty-launcher-3829-thread-1) [n:127.0.0.1:38940_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 2563677 INFO  (jetty-launcher-3829-thread-1) [n:127.0.0.1:38940_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
   [junit4]   2> 2563703 INFO  (jetty-launcher-3829-thread-2) [n:127.0.0.1:51488_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 2563704 INFO  (jetty-launcher-3829-thread-2) [n:127.0.0.1:51488_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
   [junit4]   2> 2563791 INFO  (jetty-launcher-3829-thread-1) [n:127.0.0.1:38940_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 2563841 INFO  (jetty-launcher-3829-thread-2) [n:127.0.0.1:51488_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 2563872 INFO  (jetty-launcher-3829-thread-1-EventThread) [n:127.0.0.1:38940_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 2563875 INFO  (jetty-launcher-3829-thread-2-EventThread) [n:127.0.0.1:51488_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 2564017 INFO  (jetty-launcher-3829-thread-1) [n:127.0.0.1:38940_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_6BC0DC5DB76DAA3B-001/tempDir-001/node1/.
   [junit4]   2> 2564017 INFO  (jetty-launcher-3829-thread-2) [n:127.0.0.1:51488_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_6BC0DC5DB76DAA3B-001/tempDir-001/node2/.
   [junit4]   2> 2565070 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
   [junit4]   2> 2565142 WARN  (qtp1748668418-21504) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
   [junit4]   2> 2565143 WARN  (qtp1748668418-21504) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 2565149 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565153 WARN  (qtp456834748-21506) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
   [junit4]   2> 2565154 WARN  (qtp456834748-21506) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 2565155 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565178 WARN  (qtp1748668418-21511) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 2565179 WARN  (qtp1748668418-21511) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 2565179 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565181 WARN  (qtp456834748-21507) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 2565181 WARN  (qtp456834748-21507) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 2565182 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565182 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenewFail
   [junit4]   2> 2565221 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenVerify
   [junit4]   2> 2565235 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2565237 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:32927/solr ready
   [junit4]   2> 2565254 INFO  (qtp1748668418-21500) [n:127.0.0.1:51488_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnluY5eigFaCcYSXgMDFJetDAIiDcGnRQH6Ll4r7F3vBXd0B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2565254 INFO  (qtp1748668418-21500) [n:127.0.0.1:51488_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnluY5eigFaCcYSXgMDFJetDAIiDcGnRQH6Ll4r7F3vBXd0B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2565260 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565266 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2565268 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:32927/solr ready
   [junit4]   2> 2565270 INFO  (qtp1748668418-21511) [n:127.0.0.1:51488_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnluY5eigFaCcYSXgMDFJetDAIiDcGnRQH6Ll4r7F3vBXd0B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2565270 INFO  (qtp1748668418-21511) [n:127.0.0.1:51488_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnluY5eigFaCcYSXgMDFJetDAIiDcGnRQH6Ll4r7F3vBXd0B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2565275 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565276 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenVerify
   [junit4]   2> 2565317 INFO  (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testZNodePaths
   [junit4]   2> 2565328 INFO  (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testZNodePaths
   [junit4]   2> 2565370 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationOtherOp
   [junit4]   2> 2565375 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2565377 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:32927/solr ready
   [junit4]   2> 2565379 INFO  (qtp1748668418-21504) [n:127.0.0.1:51488_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params op=someSolrOperation&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2565379 INFO  (qtp1748668418-21504) [n:127.0.0.1:51488_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={op=someSolrOperation&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2565380 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bar&p=bar&t=test&e=1485653860346&s=4gnecq76LglOA1HZYdLAytYpqZE="; Path=/; Domain=127.0.0.1; Expires=Sv, 29-janv.-2017 01:37:40 GMT; HttpOnly". Invalid 'expires' attribute: Sv, 29-janv.-2017 01:37:40 GMT
   [junit4]   2> 2565381 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationOtherOp
   [junit4]   2> 2565407 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancelFail
   [junit4]   2> 2565429 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancelFail
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithDelegationTokens -Dtests.method=testDelegationTokenCancelFail -Dtests.seed=6BC0DC5DB76DAA3B -Dtests.slow=true -Dtests.locale=lv-LV -Dtests.timezone=America/Nassau -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.05s J1 | TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<200> but was:<404>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([6BC0DC5DB76DAA3B:37FE97767F7B8D7]:0)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:140)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:294)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2565474 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenSolrClient
   [junit4]   2> 2565482 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565485 INFO  (qtp1748668418-21514) [n:127.0.0.1:51488_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnluY9bigFaCcYTWwYDFP39RMI6atv1PUJAu1kQf8-E1cIDB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2565485 INFO  (qtp1748668418-21514) [n:127.0.0.1:51488_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnluY9bigFaCcYTWwYDFP39RMI6atv1PUJAu1kQf8-E1cIDB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2565486 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenSolrClient
   [junit4]   2> 2565537 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
   [junit4]   2> 2565574 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2565576 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:32927/solr ready
   [junit4]   2> 2565578 INFO  (qtp1748668418-21513) [n:127.0.0.1:51488_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnluY-bigFaCcYTmwcDFJhqRPCn9KFIBZbHVJKhnX1J407zB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2565579 INFO  (qtp1748668418-21513) [n:127.0.0.1:51488_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnluY-bigFaCcYTmwcDFJhqRPCn9KFIBZbHVJKhnX1J407zB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2565582 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159e5b98729000e, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2565583 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565587 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2565588 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:32927/solr ready
   [junit4]   2> 2565589 INFO  (qtp1748668418-21504) [n:127.0.0.1:51488_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnluY-bigFaCcYTmwcDFJhqRPCn9KFIBZbHVJKhnX1J407zB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2565589 INFO  (qtp1748668418-21504) [n:127.0.0.1:51488_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnluY-bigFaCcYTmwcDFJhqRPCn9KFIBZbHVJKhnX1J407zB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2565594 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159e5b98729000f, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2565595 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565622 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2565623 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:32927/solr ready
   [junit4]   2> 2565625 INFO  (qtp1748668418-21501) [n:127.0.0.1:51488_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnluY_UigFaCcYT1AgDFPTVR_SwwkfBLuWrJEGfjGWn-7cyB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2565625 INFO  (qtp1748668418-21501) [n:127.0.0.1:51488_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnluY_UigFaCcYT1AgDFPTVR_SwwkfBLuWrJEGfjGWn-7cyB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2565628 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565631 INFO  (qtp456834748-21505) [n:127.0.0.1:38940_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnluY_UigFaCcYT1AgDFPTVR_SwwkfBLuWrJEGfjGWn-7cyB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2565631 INFO  (qtp456834748-21505) [n:127.0.0.1:38940_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnluY_UigFaCcYT1AgDFPTVR_SwwkfBLuWrJEGfjGWn-7cyB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2565634 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565654 INFO  (qtp1748668418-21513) [n:127.0.0.1:51488_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnluY_6igFaCcYT-gkDFDxbXDp0mylegbEts3YCcy2PqssDB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2565654 INFO  (qtp1748668418-21513) [n:127.0.0.1:51488_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnluY_6igFaCcYT-gkDFDxbXDp0mylegbEts3YCcy2PqssDB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2565659 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2565660 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:32927/solr ready
   [junit4]   2> 2565662 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565664 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159e5b987290011, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2565665 INFO  (qtp456834748-21505) [n:127.0.0.1:38940_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnluY_6igFaCcYT-gkDFDxbXDp0mylegbEts3YCcy2PqssDB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2565665 INFO  (qtp456834748-21505) [n:127.0.0.1:38940_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnluY_6igFaCcYT-gkDFDxbXDp0mylegbEts3YCcy2PqssDB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2565670 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2565671 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:32927/solr ready
   [junit4]   2> 2565673 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565675 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159e5b987290012, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2565692 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2565694 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:32927/solr ready
   [junit4]   2> 2565695 INFO  (qtp1748668418-21514) [n:127.0.0.1:51488_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnluZAkigFaCcYUJAoDFL2rsR0vm1uWShWV704kmBC25ayGB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2565695 INFO  (qtp1748668418-21514) [n:127.0.0.1:51488_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnluZAkigFaCcYUJAoDFL2rsR0vm1uWShWV704kmBC25ayGB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2565699 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565700 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159e5b987290013, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2565701 INFO  (qtp456834748-21509) [n:127.0.0.1:38940_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVnluZAkigFaCcYUJAoDFL2rsR0vm1uWShWV704kmBC25ayGB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2565701 INFO  (qtp456834748-21509) [n:127.0.0.1:38940_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVnluZAkigFaCcYUJAoDFL2rsR0vm1uWShWV704kmBC25ayGB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 2565707 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2565708 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:32927/solr ready
   [junit4]   2> 2565710 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565712 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
   [junit4]   2> 2565742 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancel
   [junit4]   2> 2565758 ERROR (qtp1748668418-21514) [n:127.0.0.1:51488_solr    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_11]
   [junit4]   2> 2565766 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2565767 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:32927/solr ready
   [junit4]   2> 2565769 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565772 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159e5b987290015, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2565774 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565789 ERROR (qtp456834748-21505) [n:127.0.0.1:38940_solr    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_12]
   [junit4]   2> 2565795 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2565796 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:32927/solr ready
   [junit4]   2> 2565798 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565800 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159e5b987290016, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2565802 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=C, 01-janv.-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: C, 01-janv.-1970 00:00:00 GMT
   [junit4]   2> 2565802 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[6BC0DC5DB76DAA3B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancel
   [junit4]   2> 2565804 INFO  (jetty-launcher-3829-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@131871a3{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2565804 INFO  (jetty-launcher-3829-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@7612f839{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2565805 INFO  (jetty-launcher-3829-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1133862698
   [junit4]   2> 2565805 INFO  (jetty-launcher-3829-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1007371101
   [junit4]   2> 2565816 INFO  (jetty-launcher-3829-thread-1) [    ] o.a.s.c.Overseer Overseer (id=97361451964497926-127.0.0.1:51488_solr-n_0000000000) closing
   [junit4]   2> 2565816 INFO  (OverseerStateUpdate-97361451964497926-127.0.0.1:51488_solr-n_0000000000) [n:127.0.0.1:51488_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:51488_solr
   [junit4]   2> 2565821 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159e5b987290006, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2565821 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159e5b987290005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2567321 WARN  (zkCallback-3840-thread-3-processing-n:127.0.0.1:51488_solr) [n:127.0.0.1:51488_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2567321 WARN  (zkCallback-3839-thread-2-processing-n:127.0.0.1:38940_solr) [n:127.0.0.1:38940_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2567322 INFO  (jetty-launcher-3829-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 2567322 INFO  (jetty-launcher-3829-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 2567323 ERROR (Thread[Thread-4359,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:38940_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 2567323 ERROR (Thread[Thread-4357,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:51488_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 2567336 INFO  (jetty-launcher-3829-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@499ae4e6{/solr,null,UNAVAILABLE}
   [junit4]   2> 2567339 INFO  (jetty-launcher-3829-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7d1b27b7{/solr,null,UNAVAILABLE}
   [junit4]   2> 2567342 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[6BC0DC5DB76DAA3B]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:32927 32927
   [junit4]   2> 2567442 INFO  (Thread-4340) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:32927 32927
   [junit4]   2> 2567443 WARN  (Thread-4340) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	14	/solr/aliases.json
   [junit4]   2> 	14	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	14	/solr/clusterstate.json
   [junit4]   2> 	4	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_7
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_6
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_9
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_8
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_5
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_12
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_11
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_10
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	14	/solr/live_nodes
   [junit4]   2> 	14	/solr/collections
   [junit4]   2> 	3	/solr/overseer/queue
   [junit4]   2> 	3	/solr/overseer/collection-queue-work
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 
   [junit4]   2> 2567444 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[6BC0DC5DB76DAA3B]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_6BC0DC5DB76DAA3B-001
   [junit4]   2> Jan 28, 2017 3:37:42 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=780, maxMBSortInHeap=7.059648126202074, sim=RandomSimilarity(queryNorm=true): {}, locale=lv-LV, timezone=America/Nassau
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=73420672,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [CdcrVersionReplicationTest, TestOnReconnectListenerSupport, TestMissingGroups, CoreAdminHandlerTest, TestLMDirichletSimilarityFactory, SolrXmlInZkTest, DistributedFacetPivotSmallAdvancedTest, TestSolrIndexConfig, SuggesterTSTTest, TestFuzzyAnalyzedSuggestions, LoggingHandlerTest, PeerSyncWithIndexFingerprintCachingTest, FullSolrCloudDistribCmdsTest, TestFoldingMultitermQuery, TestSQLHandlerNonCloud, TestCodecSupport, JSONWriterTest, DistributedFacetPivotWhiteBoxTest, TestSystemIdResolver, TestCloudRecovery, TestUnifiedSolrHighlighter, TestCrossCoreJoin, ExitableDirectoryReaderTest, TestTestInjection, ShardRoutingCustomTest, TestReplicationHandlerBackup, ResourceLoaderTest, TestConfigReload, ChaosMonkeySafeLeaderTest, TestLegacyFieldCache, SecurityConfHandlerTest, ForceLeaderTest, ShardSplitTest, RAMDirectoryFactoryTest, ResponseLogComponentTest, SpellingQueryConverterTest, ZkSolrClientTest, ConfigSetsAPITest, SpellCheckComponentTest, SuggesterTest, TestManagedSynonymFilterFactory, SpellPossibilityIteratorTest, TestExactSharedStatsCache, DistanceUnitsTest, AliasIntegrationTest, TestPointFields, CircularListTest, SolrIndexSplitterTest, HighlighterConfigTest, DistributedFacetPivotLargeTest, TestGraphMLResponseWriter, TemplateUpdateProcessorTest, TestConfigOverlay, OutOfBoxZkACLAndCredentialsProvidersTest, TestConfigSetsAPIExclusivity, ExternalFileFieldSortTest, TestSimpleTrackingShardHandler, CleanupOldIndexTest, CoreSorterTest, TestDistributedStatsComponentCardinality, UUIDFieldTest, CdcrUpdateLogTest, MinimalSchemaTest, TestRandomFaceting, PreAnalyzedFieldManagedSchemaCloudTest, BitVectorTest, DistributedIntervalFacetingTest, TestRTimerTree, TestSchemalessBufferedUpdates, TestAnalyzedSuggestions, TestRequestStatusCollectionAPI, DistributedTermsComponentTest, TestSchemaResource, TestDefaultStatsCache, StressHdfsTest, MetricUtilsTest, TestLMJelinekMercerSimilarityFactory, BlockDirectoryTest, TestUseDocValuesAsStored2, TestComplexPhraseLeadingWildcard, DistributedExpandComponentTest, TestDynamicLoading, TestRawTransformer, TestCustomSort, TestStressRecovery, TestReplicaProperties, TestStressLiveNodes, InfoHandlerTest, TestManagedSchemaAPI, TestJmxMonitoredMap, DateRangeFieldTest, TestFastWriter, AddBlockUpdateTest, AtomicUpdatesTest, URLClassifyProcessorTest, TestDocSet, TestZkChroot, CdcrReplicationHandlerTest, HdfsChaosMonkeyNothingIsSafeTest, TestConfig, DeleteInactiveReplicaTest, TestPushWriter, TestSolrDynamicMBean, SuggestComponentContextFilterQueryTest, QueryParsingTest, ReplaceNodeTest, TestChildDocTransformer, NotRequiredUniqueKeyTest, TestSegmentSorting, OverseerTaskQueueTest, TestRecovery, BlockJoinFacetSimpleTest, BadIndexSchemaTest, TriLevelCompositeIdRoutingTest, TestSolrDeletionPolicy2, TestExclusionRuleCollectionAccess, SolrCmdDistributorTest, CacheHeaderTest, DistribCursorPagingTest, TestDocBasedVersionConstraints, SSLMigrationTest, DeleteNodeTest, TestSizeLimitedDistributedMap, ReturnFieldsTest, SharedFSAutoReplicaFailoverUtilsTest, CollectionTooManyReplicasTest, TestCollectionAPI, WordBreakSolrSpellCheckerTest, TestRestoreCore, TestSolr4Spatial2, RegexBoostProcessorTest, TestFileDictionaryLookup, TestUtils, BasicDistributedZk2Test, AutoCommitTest, CollectionsAPISolrJTest, TestXmlQParser, TestValueSourceCache, TestLeaderInitiatedRecoveryThread, TestInfoStreamLogging, FieldMutatingUpdateProcessorTest, TestSimpleQParserPlugin, LeaderElectionContextKeyTest, BasicFunctionalityTest, TestFieldCacheReopen, TestDefaultSearchFieldResource, CSVRequestHandlerTest, TestDFISimilarityFactory, TestDistributedMissingSort, TestMinMaxOnMultiValuedField, VersionInfoTest, RequestHandlersTest, TestRandomCollapseQParserPlugin, OverseerRolesTest, TestTolerantUpdateProcessorRandomCloud, TestJsonRequest, TestEmbeddedSolrServerConstructors, HLLUtilTest, TestBulkSchemaConcurrent, TestStressCloudBlindAtomicUpdates, JavabinLoaderTest, DeleteShardTest, TestSortByMinMaxFunction, TolerantUpdateProcessorTest, TestIndexingPerformance, CollectionsAPIDistributedZkTest, LeaderElectionTest, TestDistributedGrouping, TestHashPartitioner, TestRealTimeGet, TestStressReorder, TestReload, TestRangeQuery, QueryElevationComponentTest, PeerSyncTest, ConvertedLegacyTest, TestSort, TestFunctionQuery, TestIndexSearcher, SoftAutoCommitTest, SuggesterFSTTest, DocValuesTest, DistanceFunctionTest, DebugComponentTest, TestSurroundQueryParser, DisMaxRequestHandlerTest, TestQueryUtils, StandardRequestHandlerTest, TestWriterPerf, TestQueryTypes, FileBasedSpellCheckerTest, DocumentBuilderTest, TermVectorComponentTest, TestSolrQueryParser, SolrPluginUtilsTest, UpdateRequestProcessorFactoryTest, BinaryUpdateRequestHandlerTest, TestPHPSerializedResponseWriter, SOLR749Test, AlternateDirectoryTest, BadComponentTest, MultiTermTest, TestSearchPerf, OutputWriterTest, TestSolrCoreProperties, TestXIncludeConfig, TestDFRSimilarityFactory, TestBM25SimilarityFactory, ScriptEngineTest, ChaosMonkeyNothingIsSafeTest, CollectionReloadTest, ConcurrentDeleteAndCreateCollectionTest, ConnectionManagerTest, CustomCollectionTest, DistribJoinFromCollectionTest, DocValuesNotIndexedTest, LeaderFailureAfterFreshStartTest, OverseerCollectionConfigSetProcessorTest, RecoveryAfterSoftCommitTest, SharedFSAutoReplicaFailoverTest, SimpleCollectionCreateDeleteTest, SolrCloudExampleTest, TestClusterProperties, TestConfigSetsAPIZkFailure, TestCryptoKeys, TestDistribDocBasedVersion, TestLeaderElectionZkExpiry, TestRandomRequestDistribution, TestRequestForwarding, TestSSLRandomization, TestSolrCloudWithDelegationTokens]
   [junit4] Completed [521/684 (1!)] on J1 in 4.86s, 8 tests, 1 failure <<< FAILURES!

[...truncated 62977 lines...]