You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/08/23 08:23:25 UTC

[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 129 - Still Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/129/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC

3 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:     1) Thread[id=7278, name=jetty-launcher-1266-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   1) Thread[id=7278, name=jetty-launcher-1266-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
	at __randomizedtesting.SeedInfo.seed([2EB2598FEDFA575D]:0)


FAILED:  org.apache.solr.cloud.LeaderElectionContextKeyTest.test

Error Message:


Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([2EB2598FEDFA575D:A6E6665543063AA5]:0)
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertTrue(Assert.java:54)
	at org.apache.solr.cloud.LeaderElectionContextKeyTest.test(LeaderElectionContextKeyTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter

Error Message:


Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([2EB2598FEDFA575D:715674B886F6C418]:0)
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertTrue(Assert.java:54)
	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:84)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11397 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_2EB2598FEDFA575D-001/init-core-data-001
   [junit4]   2> 1024656 WARN  (SUITE-TestSolrCloudWithSecureImpersonation-seed#[2EB2598FEDFA575D]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4 numCloses=4
   [junit4]   2> 1024715 ERROR (Thread-1528) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1024840 ERROR (jetty-launcher-1266-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1024844 ERROR (jetty-launcher-1266-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1024909 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e0df3453e0005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1025156 WARN  (jetty-launcher-1266-thread-2) [n:127.0.0.1:58031_solr    ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
   [junit4]   2> 1025188 WARN  (jetty-launcher-1266-thread-1) [n:127.0.0.1:61782_solr    ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
   [junit4]   2> 1025423 WARN  (jetty-launcher-1266-thread-1) [n:127.0.0.1:61782_solr    ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
   [junit4]   2> 1025425 WARN  (jetty-launcher-1266-thread-2) [n:127.0.0.1:58031_solr    ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
   [junit4]   2> 1025713 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[2EB2598FEDFA575D]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=test&e=1503508601271&s=mqIOfPv6xJ6R86Fu5qPvFbRROto="; Path=/; Domain=127.0.0.1; Expires= , 23-   -2017 17:16:41 GMT; HttpOnly". Invalid 'expires' attribute: , 23- -2017 17:16:41 GMT
   [junit4]   2> 1025736 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[2EB2598FEDFA575D]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bar&p=bar&t=test&e=1503508601399&s=c3IapSpDySA1eu9NEt9rbBG35Ss="; Path=/; Domain=127.0.0.1; Expires= , 23-   -2017 17:16:41 GMT; HttpOnly". Invalid 'expires' attribute: , 23- -2017 17:16:41 GMT
   [junit4]   2> 1025758 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[2EB2598FEDFA575D]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostUsersGroup&p=anyHostUsersGroup&t=test&e=1503508601422&s=xg8UwLdGIsu0n3jzXTVZb1hEfHw="; Path=/; Domain=127.0.0.1; Expires= , 23-   -2017 17:16:41 GMT; HttpOnly". Invalid 'expires' attribute: , 23- -2017 17:16:41 GMT
   [junit4]   2> 1025777 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[2EB2598FEDFA575D]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=test&e=1503508601444&s=ltUeOk9pJbDoQKiI2jF/8FPNuIg="; Path=/; Domain=127.0.0.1; Expires= , 23-   -2017 17:16:41 GMT; HttpOnly". Invalid 'expires' attribute: , 23- -2017 17:16:41 GMT
   [junit4]   2> 1025798 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[2EB2598FEDFA575D]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=test&e=1503508601460&s=Ov3PxumbyaPHDaowfMol0I8vj58="; Path=/; Domain=127.0.0.1; Expires= , 23-   -2017 17:16:41 GMT; HttpOnly". Invalid 'expires' attribute: , 23- -2017 17:16:41 GMT
   [junit4]   2> 1025820 WARN  (qtp250307323-7228) [    ] o.e.j.s.ServletHandler /solr/admin/collections
   [junit4]   2> org.apache.solr.common.SolrException: Error during request authentication, 
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:457)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:213)
   [junit4]   2> 	at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> 	at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
   [junit4]   2> 	at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:134)
   [junit4]   2> 	at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
   [junit4]   2> 	at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:243)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
   [junit4]   2> 	... 26 more
   [junit4]   2> 1025840 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[2EB2598FEDFA575D]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noGroups&p=noGroups&t=test&e=1503508601504&s=1+Y3sCS+kDRxidzeyBjzVKcKPPU="; Path=/; Domain=127.0.0.1; Expires= , 23-   -2017 17:16:41 GMT; HttpOnly". Invalid 'expires' attribute: , 23- -2017 17:16:41 GMT
   [junit4]   2> 1028533 WARN  (httpShardExecutor-3205-thread-1-processing-n:127.0.0.1:61782_solr [http:////127.0.0.1:61782//solr] http:////127.0.0.1:61782//solr) [n:127.0.0.1:61782_solr    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=$&p=$&t=test&e=1503508602459&s=rcaWwpPTrEqJZoOZdG0M/K1wEHA="; Path=/; Domain=127.0.0.1; Expires= , 23-   -2017 17:16:42 GMT; HttpOnly". Invalid 'expires' attribute: , 23- -2017 17:16:42 GMT
   [junit4]   2> 1029538 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[2EB2598FEDFA575D]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=user&p=user&t=test&e=1503508602120&s=zmj4xunVWVnLVmKMKqnLz4HjSZk="; Path=/; Domain=127.0.0.1; Expires= , 23-   -2017 17:16:42 GMT; HttpOnly". Invalid 'expires' attribute: , 23- -2017 17:16:42 GMT
   [junit4]   2> 1029546 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[2EB2598FEDFA575D]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=user&p=user&t=test&e=1503508605212&s=dqQU+SQ5ZI44CRQ1aDvXjvzNOIY="; Path=/; Domain=127.0.0.1; Expires= , 23-   -2017 17:16:45 GMT; HttpOnly". Invalid 'expires' attribute: , 23- -2017 17:16:45 GMT
   [junit4]   2> 1029556 WARN  (qtp760845770-7240) [n:127.0.0.1:58031_solr    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=$&p=$&t=test&e=1503508605223&s=JD93+xdYCR2Swu44IQjH5qT114M="; Path=/; Domain=127.0.0.1; Expires= , 23-   -2017 17:16:45 GMT; HttpOnly". Invalid 'expires' attribute: , 23- -2017 17:16:45 GMT
   [junit4]   2> 1029557 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[2EB2598FEDFA575D]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=user&p=user&t=test&e=1503508605220&s=uEgFVsFCdh2RNQr5unVsVVmUYU4="; Path=/; Domain=127.0.0.1; Expires= , 23-   -2017 17:16:45 GMT; HttpOnly". Invalid 'expires' attribute: , 23- -2017 17:16:45 GMT
   [junit4]   2> 1029557 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[2EB2598FEDFA575D]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=$&p=$&t=test&e=1503508605223&s=JD93+xdYCR2Swu44IQjH5qT114M="; Path=/; Domain=127.0.0.1; Expires= , 23-   -2017 17:16:45 GMT; HttpOnly". Invalid 'expires' attribute: , 23- -2017 17:16:45 GMT
   [junit4]   2> 1044638 WARN  (jetty-launcher-1266-thread-2-SendThread(127.0.0.1:56373)) [n:127.0.0.1:58031_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15084ms for sessionid 0x15e0df3453e0004
   [junit4]   2> 1044638 WARN  (jetty-launcher-1266-thread-2-SendThread(127.0.0.1:56373)) [n:127.0.0.1:58031_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 19153ms for sessionid 0x15e0df3453e0007
   [junit4]   2> 1044639 WARN  (jetty-launcher-1266-thread-1-SendThread(127.0.0.1:56373)) [n:127.0.0.1:61782_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 19170ms for sessionid 0x15e0df3453e0008
   [junit4]   2> 1044639 WARN  (jetty-launcher-1266-thread-1-SendThread(127.0.0.1:56373)) [n:127.0.0.1:61782_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 16000ms for sessionid 0x15e0df3453e0006
   [junit4]   2> 1044639 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e0df3453e0007, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1044639 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e0df3453e0008, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1044640 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e0df3453e0004, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1044640 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e0df3453e0006, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1044687 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[2EB2598FEDFA575D]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=wrongHost&p=wrongHost&t=test&e=1503508605266&s=qccZWUjxAZUIE+ELizaB/JbiJVw="; Path=/; Domain=127.0.0.1; Expires= , 23-   -2017 17:16:45 GMT; HttpOnly". Invalid 'expires' attribute: , 23- -2017 17:16:45 GMT
   [junit4]   2> 1044764 WARN  (zkCallback-1274-thread-1-processing-n:127.0.0.1:58031_solr) [n:127.0.0.1:58031_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4cc7d9f6 name: ZooKeeperConnection Watcher:127.0.0.1:56373/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1044764 WARN  (zkCallback-1274-thread-1-processing-n:127.0.0.1:58031_solr) [n:127.0.0.1:58031_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 1044793 WARN  (zkCallback-1285-thread-3) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@78fe087c name: ZooKeeperConnection Watcher:127.0.0.1:56373/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1044793 WARN  (zkCallback-1285-thread-3) [    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 1044795 WARN  (zkCallback-1279-thread-7-processing-n:127.0.0.1:61782_solr) [n:127.0.0.1:61782_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@21fcea36 name: ZooKeeperConnection Watcher:127.0.0.1:56373/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1044795 WARN  (zkCallback-1279-thread-7-processing-n:127.0.0.1:61782_solr) [n:127.0.0.1:61782_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 1044859 WARN  (qtp250307323-7234) [n:127.0.0.1:61782_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.h.s.ShellBasedUnixGroupsMapping got exception trying to get groups for user bar: id: invalid user name: "bar"
   [junit4]   2> id: invalid user name: "bar"
   [junit4]   2> 
   [junit4]   2> 1044861 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[2EB2598FEDFA575D]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bogusGroup&p=bogusGroup&t=test&e=1503508620396&s=tx/wUHWjhq6InJjoy5cQKQO2W7g="; Path=/; Domain=127.0.0.1; Expires= , 23-   -2017 17:17:00 GMT; HttpOnly". Invalid 'expires' attribute: , 23- -2017 17:17:00 GMT
   [junit4]   2> 1045094 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[2EB2598FEDFA575D]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostAnyUser&p=anyHostAnyUser&t=test&e=1503508620760&s=VDyVqoLkMih+54pmHzbcONLvWEs="; Path=/; Domain=127.0.0.1; Expires= , 23-   -2017 17:17:00 GMT; HttpOnly". Invalid 'expires' attribute: , 23- -2017 17:17:00 GMT
   [junit4]   2> 1045157 WARN  (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[2EB2598FEDFA575D]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noHosts&p=noHosts&t=test&e=1503508620815&s=PHc/tJrzxoGnMoRMOBNjQ/KUyQM="; Path=/; Domain=127.0.0.1; Expires= , 23-   -2017 17:17:00 GMT; HttpOnly". Invalid 'expires' attribute: , 23- -2017 17:17:00 GMT
   [junit4]   2> 1046118 ERROR (Thread[Thread-1549,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:58031_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 1046435 WARN  (jetty-launcher-1266-thread-1-SendThread(127.0.0.1:56373)) [n:127.0.0.1:61782_solr    ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x15e0df3453e0008 has expired
   [junit4]   2> 1046985 WARN  (zkCallback-1279-thread-1-processing-n:127.0.0.1:61782_solr) [n:127.0.0.1:61782_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1048478 WARN  (zkCallback-1279-thread-13-processing-n:127.0.0.1:61782_solr) [n:127.0.0.1:61782_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /collections/forwardingCollection/state.json]
   [junit4]   2> 1048480 WARN  (zkCallback-1279-thread-10-processing-n:127.0.0.1:61782_solr) [n:127.0.0.1:61782_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /collections/forwardingCollection/state.json]
   [junit4]   2> 1048481 ERROR (Thread[Thread-1547,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:61782_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 1048494 ERROR (SUITE-TestSolrCloudWithSecureImpersonation-seed#[2EB2598FEDFA575D]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1056012 WARN  (Thread-1528) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/collections/forwardingCollection/state.json
   [junit4]   2> 	4	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	3	/solr/collections
   [junit4]   2> 	3	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 
   [junit4]   2> Aug 23, 2017 7:17:11 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> 1074913 ERROR (jetty-launcher-1266-thread-2-EventThread) [n:127.0.0.1:58031_solr    ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
   [junit4]   2> java.lang.IllegalStateException: Client is not started
   [junit4]   2> 	at com.google.common.base.Preconditions.checkState(Preconditions.java:149)
   [junit4]   2> 	at org.apache.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:113)
   [junit4]   2> 	at org.apache.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:477)
   [junit4]   2> 	at org.apache.curator.framework.imps.NamespaceFacade.getZooKeeper(NamespaceFacade.java:112)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl$4.call(GetDataBuilderImpl.java:302)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl$4.call(GetDataBuilderImpl.java:291)
   [junit4]   2> 	at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:107)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2> 	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> Aug 23, 2017 7:17:31 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   [junit4]   2>    1) Thread[id=7278, name=jetty-launcher-1266-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2>         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2>         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2>         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2>         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> Aug 23, 2017 7:17:31 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=7278, name=jetty-launcher-1266-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]   2> 1076238 ERROR (jetty-launcher-1266-thread-1-EventThread) [n:127.0.0.1:61782_solr    ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2> 	at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2> 	at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2> 	at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2> 	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> 1076239 WARN  (jetty-launcher-1266-thread-1-EventThread) [n:127.0.0.1:61782_solr    ] o.a.c.ConnectionState Session expired event received
   [junit4]   2> 1076240 WARN  (jetty-launcher-1266-thread-1-SendThread(127.0.0.1:56373)) [n:127.0.0.1:61782_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
   [junit4]   2> Aug 23, 2017 7:17:31 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=979, maxMBSortInHeap=7.152869178685901, sim=RandomSimilarity(queryNorm=true): {}, locale=ar-BH, timezone=Europe/Gibraltar
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=3,free=278095784,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrCloudSnapshots, SoftAutoCommitTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, OverseerTest, TestFaceting, TestNRTOpen, ReturnFieldsTest, RegexBoostProcessorTest, DistanceFunctionTest, TestOverriddenPrefixQueryForCustomFieldType, UnloadDistributedZkTest, SuggesterWFSTTest, TestLegacyField, DistributedFacetPivotSmallTest, TestJsonRequest, TestTlogReplica, TestSolrConfigHandler, TestRestManager, TestUpdate, AddSchemaFieldsUpdateProcessorFactoryTest, TestPHPSerializedResponseWriter, BlobRepositoryCloudTest, HdfsTlogReplayBufferedWhileIndexingTest, OutputWriterTest, TestFieldTypeResource, ZkStateWriterTest, TestPushWriter, LeaderInitiatedRecoveryOnCommitTest, SortSpecParsingTest, TestGraphTermsQParserPlugin, TestConfigsApi, TestTrackingShardHandlerFactory, MetricsConfigTest, TestMaxScoreQueryParser, TestXIncludeConfig, RemoteQueryErrorTest, HLLUtilTest, HardAutoCommitTest, TestTolerantSearch, TestTolerantUpdateProcessorRandomCloud, TestMinMaxOnMultiValuedField, TestMultiWordSynonyms, RecoveryZkTest, CollectionStateFormat2Test, TestLockTree, TestInitQParser, TimeZoneUtilsTest, TestSolrFieldCacheBean, BitVectorTest, OverseerStatusTest, TestSortByMinMaxFunction, ConnectionReuseTest, DistributedQueryComponentCustomSortTest, AnalyticsMergeStrategyTest, AliasIntegrationTest, TestRandomFlRTGCloud, TestCursorMarkWithoutUniqueKey, TestDefaultStatsCache, TestStressVersions, TestSolrCLIRunExample, NodeMutatorTest, QueryResultKeyTest, DocValuesTest, TestPhraseSuggestions, DirectUpdateHandlerOptimizeTest, SaslZkACLProviderTest, TestManagedSynonymFilterFactory, TestReloadDeadlock, BasicAuthStandaloneTest, TestCollationFieldDocValues, DirectUpdateHandlerTest, TestConfigSetsAPIExclusivity, SpatialFilterTest, DateMathParserTest, OverseerModifyCollectionTest, TestIBSimilarityFactory, TestDynamicLoading, TestJsonFacetRefinement, CircularListTest, TestStressReorder, TestRandomDVFaceting, TestPseudoReturnFields, CSVRequestHandlerTest, ScriptEngineTest, TestShortCircuitedRequests, TestLegacyNumericUtils, QueryParsingTest, TestSSLRandomization, MoveReplicaTest, SegmentsInfoRequestHandlerTest, TestCloudInspectUtil, DateRangeFieldTest, HdfsNNFailoverTest, TestAtomicUpdateErrorCases, TestSolrCloudWithSecureImpersonation]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=2EB2598FEDFA575D -Dtests.slow=true -Dtests.locale=ar-BH -Dtests.timezone=Europe/Gibraltar -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J0 | TestSolrCloudWithSecureImpersonation (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 
   [junit4]    >    1) Thread[id=7278, name=jetty-launcher-1266-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2EB2598FEDFA575D]:0)
   [junit4] Completed [216/730 (1!)] on J0 in 51.60s, 12 tests, 1 error <<< FAILURES!

[...truncated 960 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderElectionContextKeyTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionContextKeyTest_2EB2598FEDFA575D-001/init-core-data-001
   [junit4]   2> 2858610 WARN  (SUITE-LeaderElectionContextKeyTest-seed#[2EB2598FEDFA575D]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=37 numCloses=37
   [junit4]   2> 2858621 ERROR (Thread-6330) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2858767 ERROR (jetty-launcher-3284-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2859394 WARN  (jetty-launcher-1266-thread-1-SendThread(127.0.0.1:56373)) [n:127.0.0.1:61782_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
   [junit4]   2> 2860495 WARN  (jetty-launcher-1266-thread-1-SendThread(127.0.0.1:56373)) [n:127.0.0.1:61782_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
   [junit4]   2> 2861600 WARN  (jetty-launcher-1266-thread-1-SendThread(127.0.0.1:56373)) [n:127.0.0.1:61782_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
   [junit4]   2> 2862707 WARN  (jetty-launcher-1266-thread-1-SendThread(127.0.0.1:56373)) [n:127.0.0.1:61782_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
   [junit4]   2> 2863812 WARN  (jetty-launcher-1266-thread-1-SendThread(127.0.0.1:56373)) [n:127.0.0.1:61782_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderElectionContextKeyTest -Dtests.method=test -Dtests.seed=2EB2598FEDFA575D -Dtests.slow=true -Dtests.locale=cs -Dtests.timezone=America/Denver -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 0.17s J0 | LeaderElectionContextKeyTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2EB2598FEDFA575D:A6E6665543063AA5]:0)
   [junit4]    > 	at org.apache.solr.cloud.LeaderElectionContextKeyTest.test(LeaderElectionContextKeyTest.java:88)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2864859 ERROR (SUITE-LeaderElectionContextKeyTest-seed#[2EB2598FEDFA575D]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2875539 WARN  (jetty-launcher-1266-thread-1-SendThread(127.0.0.1:56373)) [n:127.0.0.1:61782_solr    ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
   [junit4]   2> 2875546 WARN  (Thread-6330) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/aliases.json
   [junit4]   2> 	2	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/collections/testCollection2/state.json
   [junit4]   2> 	5	/solr/collections/testCollection1/state.json
   [junit4]   2> 	2	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/live_nodes
   [junit4]   2> 	2	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderElectionContextKeyTest_2EB2598FEDFA575D-001
   [junit4]   2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=true): {}, locale=cs, timezone=America/Denver
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=153542384,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrCloudSnapshots, SoftAutoCommitTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, OverseerTest, TestFaceting, TestNRTOpen, ReturnFieldsTest, RegexBoostProcessorTest, DistanceFunctionTest, TestOverriddenPrefixQueryForCustomFieldType, UnloadDistributedZkTest, SuggesterWFSTTest, TestLegacyField, DistributedFacetPivotSmallTest, TestJsonRequest, TestTlogReplica, TestSolrConfigHandler, TestRestManager, TestUpdate, AddSchemaFieldsUpdateProcessorFactoryTest, TestPHPSerializedResponseWriter, BlobRepositoryCloudTest, HdfsTlogReplayBufferedWhileIndexingTest, OutputWriterTest, TestFieldTypeResource, ZkStateWriterTest, TestPushWriter, LeaderInitiatedRecoveryOnCommitTest, SortSpecParsingTest, TestGraphTermsQParserPlugin, TestConfigsApi, TestTrackingShardHandlerFactory, MetricsConfigTest, TestMaxScoreQueryParser, TestXIncludeConfig, RemoteQueryErrorTest, HLLUtilTest, HardAutoCommitTest, TestTolerantSearch, TestTolerantUpdateProcessorRandomCloud, TestMinMaxOnMultiValuedField, TestMultiWordSynonyms, RecoveryZkTest, CollectionStateFormat2Test, TestLockTree, TestInitQParser, TimeZoneUtilsTest, TestSolrFieldCacheBean, BitVectorTest, OverseerStatusTest, TestSortByMinMaxFunction, ConnectionReuseTest, DistributedQueryComponentCustomSortTest, AnalyticsMergeStrategyTest, AliasIntegrationTest, TestRandomFlRTGCloud, TestCursorMarkWithoutUniqueKey, TestDefaultStatsCache, TestStressVersions, TestSolrCLIRunExample, NodeMutatorTest, QueryResultKeyTest, DocValuesTest, TestPhraseSuggestions, DirectUpdateHandlerOptimizeTest, SaslZkACLProviderTest, TestManagedSynonymFilterFactory, TestReloadDeadlock, BasicAuthStandaloneTest, TestCollationFieldDocValues, DirectUpdateHandlerTest, TestConfigSetsAPIExclusivity, SpatialFilterTest, DateMathParserTest, OverseerModifyCollectionTest, TestIBSimilarityFactory, TestDynamicLoading, TestJsonFacetRefinement, CircularListTest, TestStressReorder, TestRandomDVFaceting, TestPseudoReturnFields, CSVRequestHandlerTest, ScriptEngineTest, TestShortCircuitedRequests, TestLegacyNumericUtils, QueryParsingTest, TestSSLRandomization, MoveReplicaTest, SegmentsInfoRequestHandlerTest, TestCloudInspectUtil, DateRangeFieldTest, HdfsNNFailoverTest, TestAtomicUpdateErrorCases, TestSolrCloudWithSecureImpersonation, TestQuerySenderListener, TestFuzzyAnalyzedSuggestions, SharedFSAutoReplicaFailoverTest, TestQuerySenderNoQuery, TestRuleBasedAuthorizationPlugin, BlockJoinFacetDistribTest, InfixSuggestersTest, SolrTestCaseJ4Test, TestConfigSetProperties, PropertiesRequestHandlerTest, NumericFieldsTest, TestAuthenticationFramework, TestStressUserVersions, AlternateDirectoryTest, CollectionsAPIAsyncDistributedZkTest, TestInPlaceUpdatesDistrib, SimplePostToolTest, CloneFieldUpdateProcessorFactoryTest, DistributedSpellCheckComponentTest, DistanceUnitsTest, TestDistributedStatsComponentCardinality, TestXmlQParser, MergeStrategyTest, TestLeaderInitiatedRecoveryThread, UUIDFieldTest, BlockDirectoryTest, DistribDocExpirationUpdateProcessorTest, EnumFieldTest, RequiredFieldsTest, TestEmbeddedSolrServerSchemaAPI, JSONWriterTest, TestHttpShardHandlerFactory, BlockJoinFacetSimpleTest, OverriddenZkACLAndCredentialsProvidersTest, HdfsDirectoryTest, TestExclusionRuleCollectionAccess, SolrJmxReporterTest, FieldAnalysisRequestHandlerTest, TestCorePropertiesReload, MultiTermTest, DeleteLastCustomShardedReplicaTest, TestManagedSchemaThreadSafety, TestCollectionsAPIViaSolrCloudCluster, DistributedSuggestComponentTest, TestLRUStatsCache, TestLMDirichletSimilarityFactory, DateFieldTest, TestIntervalFaceting, PreAnalyzedUpdateProcessorTest, SimpleCollectionCreateDeleteTest, TestPullReplica, DistributedFacetPivotLongTailTest, SuggestComponentContextFilterQueryTest, SuggesterTest, CloudExitableDirectoryReaderTest, TestBlobHandler, MigrateRouteKeyTest, BasicDistributedZkTest, HdfsRecoveryZkTest, TestDynamicFieldResource, TestManagedResource, TestCloudRecovery, DistributedQueueTest, TestFunctionQuery, URLClassifyProcessorTest, ReplicationFactorTest, TestCryptoKeys, DocValuesNotIndexedTest, TestExportWriter, TestSchemaVersionResource, SubstringBytesRefFilterTest, ResourceLoaderTest, V2StandaloneTest, DefaultValueUpdateProcessorTest, TestStressInPlaceUpdates, AnalysisErrorHandlingTest, TestGroupingSearch, TestSolrCloudWithDelegationTokens, HighlighterConfigTest, TestCloudPivotFacet, ConjunctionSolrSpellCheckerTest, DisMaxRequestHandlerTest, TestSubQueryTransformerCrossCore, TestReplicationHandler, TestDistributedSearch, ZkControllerTest, PeerSyncTest, TestBadConfig, HighlighterTest, SolrIndexSplitterTest, AnalysisAfterCoreReloadTest, TestExtendedDismaxParser, CoreAdminHandlerTest, NoCacheHeaderTest, SolrCoreCheckLockOnStartupTest, SortByFunctionTest, TestSolrDeletionPolicy1, SolrInfoBeanTest, CacheHeaderTest, TestSurroundQueryParser, TestQueryUtils, TestWriterPerf, TestReversedWildcardFilterFactory, XmlUpdateRequestHandlerTest, DocumentBuilderTest, TermVectorComponentTest, FastVectorHighlighterTest, IndexSchemaTest, TestCollationField, TestCSVResponseWriter, UniqFieldsUpdateProcessorFactoryTest, JsonLoaderTest, BinaryUpdateRequestHandlerTest, PingRequestHandlerTest, SearchHandlerTest, TestLFUCache, EchoParamsTest, TestSweetSpotSimilarityFactory, TestLMJelinekMercerSimilarityFactory, PluginInfoTest, TestFastLRUCache, PreAnalyzedFieldTest, PrimUtilsTest, TestSuggestSpellingConverter, RAMDirectoryFactoryTest, ClusterStateTest, TestUtils, TestDocumentBuilder, SliceStateTest, SystemInfoHandlerTest, TestRTGBase, CursorPagingTest, TestCrossCoreJoin, TestHighlightDedupGrouping, AssignBackwardCompatibilityTest, AssignTest, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrBootstrapTest, CdcrReplicationDistributedZkTest, CdcrReplicationHandlerTest, CdcrRequestHandlerTest, CdcrVersionReplicationTest, CleanupOldIndexTest, CollectionReloadTest, CollectionsAPISolrJTest, CreateCollectionCleanupTest, DeleteNodeTest, DeleteShardTest, DeleteStatusTest, HttpPartitionTest, LeaderElectionContextKeyTest]
   [junit4] Completed [513/730 (2!)] on J0 in 16.99s, 1 test, 1 failure <<< FAILURES!

[...truncated 360 lines...]
   [junit4] Suite: org.apache.solr.metrics.reporters.SolrSlf4jReporterTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.metrics.reporters.SolrSlf4jReporterTest_2EB2598FEDFA575D-001/init-core-data-001
   [junit4]   2> 3589094 WARN  (SUITE-SolrSlf4jReporterTest-seed#[2EB2598FEDFA575D]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 3589095 INFO  (SUITE-SolrSlf4jReporterTest-seed#[2EB2598FEDFA575D]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 3589101 INFO  (SUITE-SolrSlf4jReporterTest-seed#[2EB2598FEDFA575D]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 3589101 INFO  (SUITE-SolrSlf4jReporterTest-seed#[2EB2598FEDFA575D]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 3589108 INFO  (TEST-SolrSlf4jReporterTest.testReporter-seed#[2EB2598FEDFA575D]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testReporter
   [junit4]   2> 3595649 INFO  (TEST-SolrSlf4jReporterTest.testReporter-seed#[2EB2598FEDFA575D]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testReporter
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SolrSlf4jReporterTest -Dtests.method=testReporter -Dtests.seed=2EB2598FEDFA575D -Dtests.slow=true -Dtests.locale=vi-VN -Dtests.timezone=America/Dominica -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 6.55s J1 | SolrSlf4jReporterTest.testReporter <<<
   [junit4]    > Throwable #1: java.lang.AssertionError
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2EB2598FEDFA575D:715674B886F6C418]:0)
   [junit4]    > 	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:84)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 3595653 INFO  (SUITE-SolrSlf4jReporterTest-seed#[2EB2598FEDFA575D]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.metrics.reporters.SolrSlf4jReporterTest_2EB2598FEDFA575D-001
   [junit4]   2> NOTE: test params are: codec=HighCompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=7, maxDocsPerChunk=6, blockSize=7), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=7, blockSize=7)), sim=RandomSimilarity(queryNorm=false): {}, locale=vi-VN, timezone=America/Dominica
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=300765600,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestCSVLoader, CdcrUpdateLogTest, FieldMutatingUpdateProcessorTest, OverseerRolesTest, CheckHdfsIndexTest, TestRemoteStreaming, TestSolrXml, TestCustomSort, SignatureUpdateProcessorFactoryTest, TestXmlQParserPlugin, TestQueryTypes, ChaosMonkeySafeLeaderTest, AutoScalingHandlerTest, SpellCheckCollatorTest, DistributedMLTComponentTest, GraphQueryTest, VMParamsZkACLAndCredentialsProvidersTest, MoreLikeThisHandlerTest, TestFieldTypeCollectionResource, TestInPlaceUpdatesStandalone, VersionInfoTest, TestSubQueryTransformer, OpenCloseCoreStressTest, ImplicitSnitchTest, SampleTest, TestManagedStopFilterFactory, ConfigureRecoveryStrategyTest, TestSubQueryTransformerDistrib, LeaderElectionIntegrationTest, TestDocSet, TestLRUCache, DistributedExpandComponentTest, TermVectorComponentDistributedTest, JvmMetricsTest, TestNoOpRegenerator, SearchHandlerTest, ReplaceNodeTest, HLLSerializationTest, AddBlockUpdateTest, TestCharFilters, ZkNodePropsTest, TestTrieFacet, TestPayloadCheckQParserPlugin, RecoveryAfterSoftCommitTest, CollectionTooManyReplicasTest, TestRandomRequestDistribution, TestLegacyNumericRangeQueryBuilder, TestSystemIdResolver, TestRecoveryHdfs, BigEndianAscendingWordSerializerTest, OverseerCollectionConfigSetProcessorTest, TestValueSourceCache, TestJettySolrRunner, DistributedQueryComponentOptimizationTest, RankQueryTest, ActionThrottleTest, TestSegmentSorting, TestDistributedMissingSort, TestPerFieldSimilarity, TestSolr4Spatial, TestMaxTokenLenTokenizer, MoveReplicaHDFSUlogDirTest, TestBackupRepositoryFactory, IndexSchemaRuntimeFieldTest, SuggesterFSTTest, TestLegacyFieldCache, ShardRoutingCustomTest, CopyFieldTest, TestExactStatsCache, TestManagedSchemaAPI, TestNumericTerms64, TestDownShardTolerantSearch, TestElisionMultitermQuery, DOMUtilTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestBM25SimilarityFactory, TestReloadAndDeleteDocs, TestSolrQueryParser, UpdateRequestProcessorFactoryTest, TestSearcherReuse, TestSimpleTrackingShardHandler, TestQueryWrapperFilter, TestReplicationHandlerBackup, TestStressLiveNodes, TestLegacyFieldReuse, TestComponentsName, TestStressCloudBlindAtomicUpdates, MissingSegmentRecoveryTest, BadComponentTest, TestCloudSearcherWarming, ExternalFileFieldSortTest, HdfsUnloadDistributedZkTest, SolrCmdDistributorTest, TestZkChroot, TestSolrCoreSnapshots, SpellingQueryConverterTest, SharedFSAutoReplicaFailoverUtilsTest, DocumentAnalysisRequestHandlerTest, PKIAuthenticationIntegrationTest, TestJsonFacets, FacetPivotSmallTest, TestDistributedGrouping, TestImplicitCoreProperties, TestCoreAdminApis, SchemaVersionSpecificBehaviorTest, TestCloudPseudoReturnFields, TestSmileRequest, HdfsDirectoryFactoryTest, CollectionsAPIDistributedZkTest, TestDocBasedVersionConstraints, TestFastOutputStream, ClassificationUpdateProcessorTest, DistributedQueryElevationComponentTest, FileUtilsTest, StatsComponentTest, TestSolrJ, TestComplexPhraseLeadingWildcard, TestPullReplicaErrorHandling, QueryEqualityTest, SuggesterTSTTest, DistribCursorPagingTest, ManagedSchemaRoundRobinCloudTest, TestBulkSchemaConcurrent, TestEmbeddedSolrServerConstructors, TestCustomDocTransformer, SuggestComponentTest, TestOrdValues, MultiThreadedOCPTest, TestAnalyzedSuggestions, TestLocalFSCloudBackupRestore, SolrJmxReporterCloudTest, TestRestoreCore, TestMergePolicyConfig, PolyFieldTest, DeleteInactiveReplicaTest, TestSchemalessBufferedUpdates, TestDeleteCollectionOnDownNodes, DistributedTermsComponentTest, TestDistribIDF, TestClassicSimilarityFactory, DocValuesMultiTest, TestStreamBody, ParsingFieldUpdateProcessorsTest, SolrCLIZkUtilsTest, TestRequestStatusCollectionAPI, DistributedFacetPivotWhiteBoxTest, SpellCheckComponentTest, FileBasedSpellCheckerTest, ChaosMonkeySafeLeaderWithPullReplicasTest, TestSolrConfigHandlerConcurrent, LeaderFailureAfterFreshStartTest, BasicDistributedZk2Test, StressHdfsTest, TestSchemaSimilarityResource, TestFieldCollectionResource, TestConfigSetsAPI, DistribJoinFromCollectionTest, TestStressLucene, TestPartialUpdateDeduplication, RestartWhileUpdatingTest, TestHdfsCloudBackupRestore, DistributedVersionInfoTest, TestRawResponseWriter, BadIndexSchemaTest, TestConfigSetImmutable, MetricsHandlerTest, DocExpirationUpdateProcessorFactoryTest, TlogReplayBufferedWhileIndexingTest, TestFieldCacheSort, TestJoin, TestTolerantUpdateProcessorCloud, TermsComponentTest, TestCollectionAPI, ResponseHeaderTest, DistributedIntervalFacetingTest, TestDynamicFieldCollectionResource, SyncSliceTest, ShardRoutingTest, BasicZkTest, FullSolrCloudDistribCmdsTest, ClusterStateUpdateTest, TestRandomFaceting, LeaderElectionTest, ZkSolrClientTest, ZkCLITest, AutoCommitTest, TestRecovery, TestHashPartitioner, TestRealTimeGet, TestReload, TestRangeQuery, TestCoreContainer, SimpleFacetsTest, SolrCoreTest, QueryElevationComponentTest, ConvertedLegacyTest, TestSort, TestFiltering, BasicFunctionalityTest, TestLazyCores, TestIndexSearcher, ShowFileRequestHandlerTest, TestCoreDiscovery, SolrRequestParserTest, TestFoldingMultitermQuery, TestTrie, WordBreakSolrSpellCheckerTest, TestWordDelimiterFilterFactory, StatelessScriptUpdateProcessorFactoryTest, XsltUpdateRequestHandlerTest, DebugComponentTest, IndexBasedSpellCheckerTest, LukeRequestHandlerTest, DirectSolrSpellCheckerTest, PrimitiveFieldTypeTest, TestOmitPositions, RequestHandlersTest, PathHierarchyTokenizerFactoryTest, TestIndexingPerformance, LoggingHandlerTest, SolrPluginUtilsTest, TestJmxIntegration, MBeansHandlerTest, SOLR749Test, UpdateParamsTest, TestSolrIndexConfig, ResponseLogComponentTest, SolrIndexConfigTest, TestStressRecovery, TestSolrDeletionPolicy2, TestBinaryField, TestSearchPerf, MinimalSchemaTest, TestConfig, TestSolrCoreProperties, TestPostingsSolrHighlighter, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestLuceneMatchVersion, SpellPossibilityIteratorTest, TestCodecSupport, SynonymTokenizerTest, TestDFRSimilarityFactory, TestFastWriter, OpenExchangeRatesOrgProviderTest, ChaosMonkeyNothingIsSafeTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, ConcurrentDeleteAndCreateCollectionTest, ConfigSetsAPITest, ConnectionManagerTest, CustomCollectionTest, DeleteReplicaTest, ForceLeaderTest, LeaderInitiatedRecoveryOnShardRestartTest, MoveReplicaHDFSTest, OverseerTaskQueueTest, RollingRestartTest, SSLMigrationTest, ShardSplitTest, TestPrepRecovery, TestRequestForwarding, TestSizeLimitedDistributedMap, TriLevelCompositeIdRoutingTest, TestPolicyCloud, HdfsBasicDistributedZk2Test, HdfsBasicDistributedZkTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsChaosMonkeySafeLeaderTest, HdfsCollectionsAPIDistributedZkTest, HdfsRecoverLeaseTest, HdfsRestartWhileUpdatingTest, HdfsSyncSliceTest, HdfsThreadLeakTest, CachingDirectoryFactoryTest, CoreSorterTest, DirectoryFactoryTest, ExitableDirectoryReaderTest, TestConfigOverlay, TestConfigSets, TestCustomStream, TestConfigReload, TestReqParamsAPI, TestSQLHandler, TestSystemCollAutoCreate, CoreAdminCreateDiscoverTest, CoreAdminRequestStatusTest, SecurityConfHandlerTest, TestApiFramework, TestCollectionAPIs, DistributedDebugComponentTest, DistributedFacetExistsSmallTest, DistributedFacetPivotLargeTest, TestExpandComponent, TestPivotHelperCode, JavabinLoaderTest, HighlighterMaxOffsetTest, TestUnifiedSolrHighlighter, TestSlowCompositeReaderWrapper, UninvertDocValuesMergePolicyTest, WrapperMergePolicyFactoryTest, TestLegacyTerms, TestMultiValuedNumericRangeQuery, TestNumericRangeQuery32, TestNumericRangeQuery64, TestNumericTokenStream, SolrCoreMetricManagerTest, SolrMetricManagerTest, SolrMetricReporterTest, SolrMetricsIntegrationTest, SolrGangliaReporterTest, SolrGraphiteReporterTest, SolrSlf4jReporterTest]
   [junit4] Completed [613/730 (3!)] on J1 in 6.61s, 1 test, 1 failure <<< FAILURES!

[...truncated 43784 lines...]

[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 130 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/130/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC

3 tests failed.
FAILED:  org.apache.solr.cloud.TestPullReplica.testKillLeader

Error Message:
Replica state not updated in cluster state null Live Nodes: [127.0.0.1:58687_solr, 127.0.0.1:38715_solr] Last available state: DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/6)={   "pullReplicas":"1",   "replicationFactor":"1",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node3":{           "core":"pull_replica_test_kill_leader_shard1_replica_n1",           "base_url":"http://127.0.0.1:38715/solr",           "node_name":"127.0.0.1:38715_solr",           "state":"down",           "type":"NRT",           "leader":"true"},         "core_node4":{           "core":"pull_replica_test_kill_leader_shard1_replica_p2",           "base_url":"http://127.0.0.1:58687/solr",           "node_name":"127.0.0.1:58687_solr",           "state":"active",           "type":"PULL"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"100",   "autoAddReplicas":"false",   "nrtReplicas":"1",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Replica state not updated in cluster state
null
Live Nodes: [127.0.0.1:58687_solr, 127.0.0.1:38715_solr]
Last available state: DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/6)={
  "pullReplicas":"1",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"pull_replica_test_kill_leader_shard1_replica_n1",
          "base_url":"http://127.0.0.1:38715/solr",
          "node_name":"127.0.0.1:38715_solr",
          "state":"down",
          "type":"NRT",
          "leader":"true"},
        "core_node4":{
          "core":"pull_replica_test_kill_leader_shard1_replica_p2",
          "base_url":"http://127.0.0.1:58687/solr",
          "node_name":"127.0.0.1:58687_solr",
          "state":"active",
          "type":"PULL"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"100",
  "autoAddReplicas":"false",
  "nrtReplicas":"1",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([1E21CB467E045E61:57373FF21CBFCA37]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
	at org.apache.solr.cloud.TestPullReplica.doTestNoLeader(TestPullReplica.java:401)
	at org.apache.solr.cloud.TestPullReplica.testKillLeader(TestPullReplica.java:290)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping

Error Message:
Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.

Stack Trace:
java.lang.AssertionError: Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.
	at __randomizedtesting.SeedInfo.seed([1E21CB467E045E61:6FC5359008EF79AD]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping(UpdateRequestProcessorFactoryTest.java:101)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter

Error Message:
No 'solr.node' logs in: {numFound=0,start=0,docs=[]}

Stack Trace:
java.lang.AssertionError: No 'solr.node' logs in: {numFound=0,start=0,docs=[]}
	at __randomizedtesting.SeedInfo.seed([1E21CB467E045E61:41C5E6711508CD24]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:85)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11327 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestPullReplica
   [junit4]   2> 795394 INFO  (SUITE-TestPullReplica-seed#[1E21CB467E045E61]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_1E21CB467E045E61-001/init-core-data-001
   [junit4]   2> 795395 WARN  (SUITE-TestPullReplica-seed#[1E21CB467E045E61]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=12 numCloses=12
   [junit4]   2> 795395 INFO  (SUITE-TestPullReplica-seed#[1E21CB467E045E61]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 795396 INFO  (SUITE-TestPullReplica-seed#[1E21CB467E045E61]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 795399 ERROR (Thread-1307) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 795505 ERROR (jetty-launcher-2573-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 795506 ERROR (jetty-launcher-2573-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 795822 INFO  (TEST-TestPullReplica.testKillPullReplica-seed#[1E21CB467E045E61]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testKillPullReplica
   [junit4]   2> 801973 WARN  (zkCallback-2585-thread-2-processing-n:127.0.0.1:38715_solr) [n:127.0.0.1:38715_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 801988 ERROR (TEST-TestPullReplica.testKillPullReplica-seed#[1E21CB467E045E61]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 802177 ERROR (indexFetcher-3526-thread-1) [n:127.0.0.1:38715_solr c:pull_replica_test_kill_pull_replica s:shard1 r:core_node4 x:pull_replica_test_kill_pull_replica_shard1_replica_p2] o.a.s.h.IndexFetcher User aborted Replication
   [junit4]   2> 803473 INFO  (TEST-TestPullReplica.testKillPullReplica-seed#[1E21CB467E045E61]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testKillPullReplica
   [junit4]   2> 803489 INFO  (TEST-TestPullReplica.testKillLeader-seed#[1E21CB467E045E61]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testKillLeader
   [junit4]   2> 808177 WARN  (zkCallback-2596-thread-2-processing-n:127.0.0.1:38715_solr) [n:127.0.0.1:38715_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 898180 WARN  (TEST-TestPullReplica.testKillLeader-seed#[1E21CB467E045E61]) [    ] o.a.s.c.TestPullReplica Jetty 38715 not running, probably some bad test. Starting it
   [junit4]   2> 898183 ERROR (TEST-TestPullReplica.testKillLeader-seed#[1E21CB467E045E61]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 899869 INFO  (TEST-TestPullReplica.testKillLeader-seed#[1E21CB467E045E61]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testKillLeader
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestPullReplica -Dtests.method=testKillLeader -Dtests.seed=1E21CB467E045E61 -Dtests.slow=true -Dtests.locale=fr-LU -Dtests.timezone=Pacific/Pohnpei -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 96.4s J1 | TestPullReplica.testKillLeader <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Replica state not updated in cluster state
   [junit4]    > null
   [junit4]    > Live Nodes: [127.0.0.1:58687_solr, 127.0.0.1:38715_solr]
   [junit4]    > Last available state: DocCollection(pull_replica_test_kill_leader//collections/pull_replica_test_kill_leader/state.json/6)={
   [junit4]    >   "pullReplicas":"1",
   [junit4]    >   "replicationFactor":"1",
   [junit4]    >   "shards":{"shard1":{
   [junit4]    >       "range":"80000000-7fffffff",
   [junit4]    >       "state":"active",
   [junit4]    >       "replicas":{
   [junit4]    >         "core_node3":{
   [junit4]    >           "core":"pull_replica_test_kill_leader_shard1_replica_n1",
   [junit4]    >           "base_url":"http://127.0.0.1:38715/solr",
   [junit4]    >           "node_name":"127.0.0.1:38715_solr",
   [junit4]    >           "state":"down",
   [junit4]    >           "type":"NRT",
   [junit4]    >           "leader":"true"},
   [junit4]    >         "core_node4":{
   [junit4]    >           "core":"pull_replica_test_kill_leader_shard1_replica_p2",
   [junit4]    >           "base_url":"http://127.0.0.1:58687/solr",
   [junit4]    >           "node_name":"127.0.0.1:58687_solr",
   [junit4]    >           "state":"active",
   [junit4]    >           "type":"PULL"}}}},
   [junit4]    >   "router":{"name":"compositeId"},
   [junit4]    >   "maxShardsPerNode":"100",
   [junit4]    >   "autoAddReplicas":"false",
   [junit4]    >   "nrtReplicas":"1",
   [junit4]    >   "tlogReplicas":"0"}
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([1E21CB467E045E61:57373FF21CBFCA37]:0)
   [junit4]    > 	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
   [junit4]    > 	at org.apache.solr.cloud.TestPullReplica.doTestNoLeader(TestPullReplica.java:401)
   [junit4]    > 	at org.apache.solr.cloud.TestPullReplica.testKillLeader(TestPullReplica.java:290)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 899888 INFO  (TEST-TestPullReplica.testAddRemovePullReplica-seed#[1E21CB467E045E61]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testAddRemovePullReplica
   [junit4]   2> 904311 INFO  (TEST-TestPullReplica.testAddRemovePullReplica-seed#[1E21CB467E045E61]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testAddRemovePullReplica
   [junit4]   2> 904348 INFO  (TEST-TestPullReplica.testRemoveAllWriterReplicas-seed#[1E21CB467E045E61]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testRemoveAllWriterReplicas
   [junit4]   2> 913177 ERROR (TEST-TestPullReplica.testRemoveAllWriterReplicas-seed#[1E21CB467E045E61]) [    ] o.a.s.c.s.i.CloudSolrClient Request to collection pull_replica_test_remove_all_writer_replicas failed due to (503) org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:58687/solr/pull_replica_test_remove_all_writer_replicas: No registered leader was found after waiting for 4000ms , collection: pull_replica_test_remove_all_writer_replicas slice: shard1, retry? 0
   [junit4]   2> 917216 ERROR (TEST-TestPullReplica.testRemoveAllWriterReplicas-seed#[1E21CB467E045E61]) [    ] o.a.s.c.s.i.CloudSolrClient Request to collection pull_replica_test_remove_all_writer_replicas failed due to (503) org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:58687/solr/pull_replica_test_remove_all_writer_replicas: No registered leader was found after waiting for 4000ms , collection: pull_replica_test_remove_all_writer_replicas slice: shard1, retry? 0
   [junit4]   2> 921730 INFO  (TEST-TestPullReplica.testRemoveAllWriterReplicas-seed#[1E21CB467E045E61]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testRemoveAllWriterReplicas
   [junit4]   2> 921748 INFO  (TEST-TestPullReplica.testAddDocs-seed#[1E21CB467E045E61]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testAddDocs
   [junit4]   2> 921751 WARN  (OverseerThreadFactory-3514-thread-5-processing-n:127.0.0.1:58687_solr) [n:127.0.0.1:58687_solr    ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 4 on collection pull_replica_test_add_docs is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance.
   [junit4]   2> 927282 INFO  (TEST-TestPullReplica.testAddDocs-seed#[1E21CB467E045E61]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testAddDocs
   [junit4]   2> 927317 INFO  (TEST-TestPullReplica.testCreateDelete-seed#[1E21CB467E045E61]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testCreateDelete { seed=[1E21CB467E045E61:5313A0A88F4A651]}
   [junit4]   2> 927326 WARN  (OverseerThreadFactory-3514-thread-5-processing-n:127.0.0.1:58687_solr) [n:127.0.0.1:58687_solr    ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 4 on collection pull_replica_test_create_delete is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance.
   [junit4]   1> / (11)
   [junit4]   1>  /configs (2)
   [junit4]   1>  DATA:
   [junit4]   1>      <?xml version="1.0" encoding="UTF-8" ?>
   [junit4]   1>      <!--
   [junit4]   1>       Licensed to the Apache Software Foundation (ASF) under one or more
   [junit4]   1>       contributor license agreements.  See the NOTICE file distributed with
   [junit4]   1>       this work for additional information regarding copyright ownership.
   [junit4]   1>       The ASF licenses this file to You under the Apache License, Version 2.0
   [junit4]   1>       (the "License"); you may not use this file except in compliance with
   [junit4]   1>       the License.  You may obtain a copy of the License at
   [junit4]   1>      
   [junit4]   1>           http://www.apache.org/licenses/LICENSE-2.0
   [junit4]   1>      
   [junit4]   1>       Unless required by applicable law or agreed to in writing, software
   [junit4]   1>       distributed under the License is distributed on an "AS IS" BASIS,
   [junit4]   1>       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   [junit4]   1>       See the License for the specific language governing permissions and
   [junit4]   1>       limitations under the License.
   [junit4]   1>      -->
   [junit4]   1>      
   [junit4]   1>      <!--
   [junit4]   1>           For more details about configurations options that may appear in
   [junit4]   1>           this file, see http://wiki.apache.org/solr/SolrConfigXml.
   [junit4]   1>      -->
   [junit4]   1>      <config>
   [junit4]   1>        <!-- In all configuration below, a prefix of "solr." for class names
   [junit4]   1>             is an alias that causes solr to search appropriate packages,
   [junit4]   1>             including org.apache.solr.(search|update|request|core|analysis)
   [junit4]   1>      
   [junit4]   1>             You may also specify a fully qualified Java classname if you
   [junit4]   1>             have your own custom plugins.
   [junit4]   1>          -->
   [junit4]   1>      
   [junit4]   1>        <!-- Controls what version of Lucene various components of Solr
   [junit4]   1>             adhere to.  Generally, you want to use the latest version to
   [junit4]   1>             get all bug fixes and improvements. It is highly recommended
   [junit4]   1>             that you fully re-index after changing this setting as it can
   [junit4]   1>             affect both how text is indexed and queried.
   [junit4]   1>        -->
   [junit4]   1>        <luceneMatchVersion>7.1.0</luceneMatchVersion>
   [junit4]   1>      
   [junit4]   1>        <!-- <lib/> directives can be used to instruct Solr to load any Jars
   [junit4]   1>             identified and use them to resolve any "plugins" specified in
   [junit4]   1>             your solrconfig.xml or schema.xml (ie: Analyzers, Request
   [junit4]   1>             Handlers, etc...).
   [junit4]   1>      
   [junit4]   1>             All directories and paths are resolved relative to the
   [junit4]   1>             instanceDir.
   [junit4]   1>      
   [junit4]   1>             Please note that <lib/> directives are processed in the order
   [junit4]   1>             that they appear in your solrconfig.xml file, and are "stacked"
   [junit4]   1>             on top of each other when building a ClassLoader - so if you have
   [junit4]   1>             plugin jars with dependencies on other jars, the "lower level"
   [junit4]   1>             dependency jars should be loaded first.
   [junit4]   1>      
   [junit4]   1>             If a "./lib" directory exists in your instanceDir, all files
   [junit4]   1>             found in it are included as if you had used the following
   [junit4]   1>             syntax...
   [junit4]   1>      
   [junit4]   1>                    <lib dir="./lib" />
   [junit4]   1>          -->
   [junit4]   1>      
   [junit4]   1>        <!-- A 'dir' option by itself adds any files found in the directory
   [junit4]   1>             to the classpath, this is useful for including all jars in a
   [junit4]   1>             directory.
   [junit4]   1>      
   [junit4]   1>             When a 'regex' is specified in addition to a 'dir', only the
   [junit4]   1>             files in that directory which completely match the regex
   [junit4]   1>             (anchored on both ends) will be included.
   [junit4]   1>      
   [junit4]   1>             If a 'dir' option (with or without a regex) is used and nothing
   [junit4]   1>             is found that matches, a warning will be logged.
   [junit4]   1>      
   [junit4]   1>             The examples below can be used to load some solr-contribs along
   [junit4]   1>             with their external dependencies.
   [junit4]   1>          -->
   [junit4]   1>        <lib dir="${solr.install.dir:../../../..}/contrib/extraction/lib" regex=".*\.jar" />
   [junit4]   1>        <lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-cell-\d.*\.jar" />
   [junit4]   1>      
   [junit4]   1>        <lib dir="${solr.install.dir:../../../..}/contrib/clustering/lib/" regex=".*\.jar" />
   [junit4]   1>        <lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-clustering-\d.*\.jar" />
   [junit4]   1>      
   [junit4]   1>        <lib dir="${solr.install.dir:../../../..}/contrib/langid/lib/" regex=".*\.jar" />
   [junit4]   1>        <lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-langid-\d.*\.jar" />
   [junit4]   1>      
   [junit4]   1>        <lib dir="${solr.install.dir:../../../..}/contrib/velocity/lib" regex=".*\.jar" />
   [junit4]   1>        <lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-velocity-\d.*\.jar" />
   [junit4]   1>        <!-- an exact 'path' can be used instead of a 'dir' to specify a
   [junit4]   1>             specific jar file.  This will cause a serious error to be logged
   [junit4]   1>             if it can't be loaded.
   [junit4]   1>          -->
   [junit4]   1>        <!--
   [junit4]   1>           <lib path="../a-jar-that-does-not-exist.jar" />
   [junit4]   1>        -->
   [junit4]   1>      
   [junit4]   1>        <!-- Data Directory
   [junit4]   1>      
   [junit4]   1>             Used to specify an alternate directory to hold all index data
   [junit4]   1>             other than the default ./data under the Solr home.  If
   [junit4]   1>             replication is in use, this should match the replication
   [junit4]   1>             configuration.
   [junit4]   1>          -->
   [junit4]   1>        <dataDir>${solr.data.dir:}</dataDir>
   [junit4]   1>      
   [junit4]   1>      
   [junit4]   1>        <!-- The DirectoryFactory to use for indexes.
   [junit4]   1>      
   [junit4]   1>             solr.StandardDirectoryFactory is filesystem
   [junit4]   1>             based and tries to pick the best implementation for the current
   [junit4]   1>             JVM and platform.  solr.NRTCachingDirectoryFactory, the default,
   [junit4]   1>             wraps solr.StandardDirectoryFactory and caches small files in memory
   [junit4]   1>             for better NRT performance.
   [junit4]   1>      
   [junit4]   1>             One can force a particular implementation via solr.MMapDirectoryFactory,
   [junit4]   1>             solr.NIOFSDirectoryFactory, or solr.SimpleFSDirectoryFactory.
   [junit4]   1>      
   [junit4]   1>             solr.RAMDirectoryFactory is memory based and not persistent.
   [junit4]   1>          -->
   [junit4]   1>        <directoryFactory name="DirectoryFactory"
   [junit4]   1>                          class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
   [junit4]   1>      
   [junit4]   1>        <!-- The CodecFactory for defining the format of the inverted index.
   [junit4]   1>             The default implementation is SchemaCodecFactory, which is the official Lucene
   [junit4]   1>             index format, but hooks into the schema to provide per-field customization of
   [junit4]   1>             the postings lists and per-document values in the fieldType element
   [junit4]   1>             (postingsFormat/docValuesFormat). Note that most of the alternative implementations
   [junit4]   1>             are experimental, so if you choose to customize the index format, it's a good
   [junit4]   1>             idea to convert back to the official format e.g. via IndexWriter.addIndexes(IndexReader)
   [junit4]   1>             before upgrading to a newer version to avoid unnecessary reindexing.
   [junit4]   1>             A "compressionMode" string element can be added to <codecFactory> to choose
   [junit4]   1>             between the existing compression modes in the default codec: "BEST_SPEED" (default)
   [junit4]   1>             or "BEST_COMPRESSION".
   [junit4]   1>        -->
   [junit4]   1>        <codecFactory class="solr.SchemaCodecFactory"/>
   [junit4]   1>      
   [junit4]   1>        <!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   [junit4]   1>             Index Config - These settings control low-level behavior of indexing
   [junit4]   1>             Most example settings here show the default value, but are commented
   [junit4]   1>             out, to more easily see where customizations have been made.
   [junit4]   1>      
   [junit4]   1>             Note: This replaces <indexDefaults> and <mainIndex> from older versions
   [junit4]   1>             ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->
   [junit4]   1>        <indexConfig>
   [junit4]   1>          <!-- maxFieldLength was removed in 4.0. To get similar behavior, include a
   [junit4]   1>               LimitTokenCountFilterFactory in your fieldType definition. E.g.
   [junit4]   1>           <filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="10000"/>
   [junit4]   1>          -->
   [junit4]   1>          <!-- Maximum time to wait for a write lock (ms) for an IndexWriter. Default: 1000 -->
   [junit4]   1>          <!-- <writeLockTimeout>1000</writeLockTimeout>  -->
   [junit4]   1>      
   [junit4]   1>          <!-- Expert: Enabling compound file will use less files for the index,
   [junit4]   1>               using fewer file descriptors on the expense of performance decrease.
   [junit4]   1>               Default in Lucene is "true". Default in Solr is "false" (since 3.6) -->
   [junit4]   1>          <!-- <useCompoundFile>false</useCompoundFile> -->
   [junit4]   1>      
   [junit4]   1>          <!-- ramBufferSizeMB sets the amount of RAM that may be used by Lucene
   [junit4]   1>               indexing for buffering added documents and deletions before they are
   [junit4]   1>               flushed to the Directory.
   [junit4]   1>               maxBufferedDocs sets a limit on the number of documents buffered
   [junit4]   1>               before flushing.
   [junit4]   1>               If both ramBufferSizeMB and maxBufferedDocs is set, then
   [junit4]   1>               Lucene will flush based on whichever limit is hit first.  -->
   [junit4]   1>          <!-- <ramBufferSizeMB>100</ramBufferSizeMB> -->
   [junit4]   1>          <!-- <maxBufferedDocs>1000</maxBufferedDocs> -->
   [junit4]   1>      
   [junit4]   1>          <!-- Expert: Merge Policy
   [junit4]   1>               The Merge Policy in Lucene controls how merging of segments is done.
   [junit4]   1>               The default since Solr/Lucene 3.3 is TieredMergePolicy.
   [junit4]   1>               The default since Lucene 2.3 was the LogByteSizeMergePolicy,
   [junit4]   1>               Even older versions of Lucene used LogDocMergePolicy.
   [junit4]   1>            -->
   [junit4]   1>          <!--
   [junit4]   1>              <mergePolicyFactory class="org.apache.solr.index.TieredMergePolicyFactory">
   [junit4]   1>                <int name="maxMergeAtOnce">10</int>
   [junit4]   1>                <int name="segmentsPerTier">10</int>
   [junit4]   1>                <double name="noCFSRatio">0.1</double>
   [junit4]   1>              </mergePolicyFactory>
   [junit4]   1>            -->
   [junit4]   1>      
   [junit4]   1>          <!-- Expert: Merge Scheduler
   [junit4]   1>               The Merge Scheduler in Lucene controls how merges are
   [junit4]   1>               performed.  The ConcurrentMergeScheduler (Lucene 2.3 default)
   [junit4]   1>               can perform merges in the background using separate threads.
   [junit4]   1>               The SerialMergeScheduler (Lucene 2.2 default) does not.
   [junit4]   1>           -->
   [junit4]   1>          <!--
   [junit4]   1>             <mergeScheduler class="org.apache.lucene.index.ConcurrentMergeScheduler"/>
   [junit4]   1>             -->
   [junit4]   1>      
   [junit4]   1>          <!-- LockFactory
   [junit4]   1>      
   [junit4]   1>               This option specifies which Lucene LockFactory implementation
   [junit4]   1>               to use.
   [junit4]   1>      
   [junit4]   1>               single = SingleInstanceLockFactory - suggested for a
   [junit4]   1>                        read-only index or when there is no possibility of
   [junit4]   1>                        another process trying to modify the index.
   [junit4]   1>               native = NativeFSLockFactory - uses OS native file locking.
   [junit4]   1>                        Do not use when multiple solr webapps in the same
   [junit4]   1>                        JVM are attempting to share a single index.
   [junit4]   1>               simple = SimpleFSLockFactory  - uses a plain file for locking
   [junit4]   1>      
   [junit4]   1>               Defaults: 'native' is default for Solr3.6 and later, otherwise
   [junit4]   1>                         'simple' is the default
   [junit4]   1>      
   [junit4]   1>               More details on the nuances of each LockFactory...
   [junit4]   1>               http://wiki.apache.org/lucene-java/AvailableLockFactories
   [junit4]   1>          -->
   [junit4]   1>          <lockType>${solr.lock.type:native}</lockType>
   [junit4]   1>      
   [junit4]   1>          <!-- Commit Deletion Policy
   [junit4]   1>               Custom deletion policies can be specified here. The class must
   [junit4]   1>               implement org.apache.lucene.index.IndexDeletionPolicy.
   [junit4]   1>      
   [junit4]   1>               The default Solr IndexDeletionPolicy implementation supports
   [junit4]   1>               deleting index commit points on number of commits, age of
   [junit4]   1>               commit point and optimized status.
   [junit4]   1>      
   [junit4]   1>               The latest commit point should always be preserved regardless
   [junit4]   1>               of the criteria.
   [junit4]   1>          -->
   [junit4]   1>          <!--
   [junit4]   1>          <deletionPolicy class="solr.SolrDeletionPolicy">
   [junit4]   1>          -->
   [junit4]   1>          <!-- The number of commit points to be kept -->
   [junit4]   1>          <!-- <str name="maxCommitsToKeep">1</str> -->
   [junit4]   1>          <!-- The number of optimized commit points to be kept -->
   [junit4]   1>          <!-- <str name="maxOptimizedCommitsToKeep">0</str> -->
   [junit4]   1>          <!--
   [junit4]   1>              Delete all commit points once they have reached the given age.
   [junit4]   1>              Supports DateMathParser syntax e.g.
   [junit4]   1>            -->
   [junit4]   1>          <!--
   [junit4]   1>             <str name="maxCommitAge">30MINUTES</str>
   [junit4]   1>             <str name="maxCommitAge">1DAY</str>
   [junit4]   1>          -->
   [junit4]   1>          <!--
   [junit4]   1>          </deletionPolicy>
   [junit4]   1>          -->
   [junit4]   1>      
   [junit4]   1>          <!-- Lucene Infostream
   [junit4]   1>      
   [junit4]   1>               To aid in advanced debugging, Lucene provides an "InfoStream"
   [junit4]   1>               of detailed information when indexing.
   [junit4]   1>      
   [junit4]   1>               Setting The value to true will instruct the underlying Lucene
   [junit4]   1>               IndexWriter to write its debugging info the specified file
   [junit4]   1>            -->
   [junit4]   1>          <!-- <infoStream file="INFOSTREAM.txt">false</infoStream> -->
   [junit4]   1>        </indexConfig>
   [junit4]   1>      
   [junit4]   1>      
   [junit4]   1>        <!-- JMX
   [junit4]   1>      
   [junit4]   1>             This example enables JMX if and only if an existing MBeanServer
   [junit4]   1>             is found, use this if you want to configure JMX through JVM
   [junit4]   1>             parameters. Remove this to disable exposing Solr configuration
   [junit4]   1>             and statistics to JMX.
   [junit4]   1>      
   [junit4]   1>             For more details see http://wiki.apache.org/solr/SolrJmx
   [junit4]   1>          -->
   [junit4]   1>        <jmx />
   [junit4]   1>        <!-- If you want to connect to a particular server, specify the
   [junit4]   1>             agentId
   [junit4]   1>          -->
   [junit4]   1>        <!-- <jmx agentId="myAgent" /> -->
   [junit4]   1>        <!-- If you want to start a new MBeanServer, specify the serviceUrl -->
   [junit4]   1>        <!-- <jmx serviceUrl="service:jmx:rmi:///jndi/rmi://localhost:9999/solr"/>
   [junit4]   1>          -->
   [junit4]   1>      
   [junit4]   1>        <!-- The default high-performance update handler -->
   [junit4]   1>        <updateHandler class="solr.DirectUpdateHandler2">
   [junit4]   1>      
   [junit4]   1>          <!-- Enables a transaction log, used for real-time get, durability, and
   [junit4]   1>               and solr cloud replica recovery.  The log can grow as big as
   [junit4]   1>               uncommitted changes to the index, so use of a hard autoCommit
   [junit4]   1>               is recommended (see below).
   [junit4]   1>               "dir" - the target directory for transaction logs, defaults to the
   [junit4]   1>                      solr data directory.
   [junit4]   1>               "numVersionBuckets" - sets the number of buckets used to keep
   [junit4]   1>                      track of max version values when checking for re-ordered
   [junit4]   1>                      updates; increase this value to reduce the cost of
   [junit4]   1>                      synchronizing access to version buckets during high-volume
   [junit4]   1>                      indexing, this requires 8 bytes (long) * numVersionBuckets
   [junit4]   1>                      of heap space per Solr core.
   [junit4]   1>          -->
   [junit4]   1>          <updateLog>
   [junit4]   1>            <str name="dir">${solr.ulog.dir:}</str>
   [junit4]   1>            <int name="numVersionBuckets">${solr.ulog.numVersionBuckets:65536}</int>
   [junit4]   1>          </updateLog>
   [junit4]   1>      
   [junit4]   1>          <!-- AutoCommit
   [junit4]   1>      
   [junit4]   1>               Perform a hard commit automatically under certain conditions.
   [junit4]   1>               Instead of enabling autoCommit, consider using "commitWithin"
   [junit4]   1>               when adding documents.
   [junit4]   1>      
   [junit4]   1>               http://wiki.apache.org/solr/UpdateXmlMessages
   [junit4]   1>      
   [junit4]   1>               maxDocs - Maximum number of documents to add since the last
   [junit4]   1>                         commit before automatically triggering a new commit.
   [junit4]   1>      
   [junit4]   1>               maxTime - Maximum amount of time in ms that is allowed to pass
   [junit4]   1>                         since a document was added before automatically
   [junit4]   1>                         triggering a new commit.
   [junit4]   1>               openSearcher - if false, the commit causes recent index changes
   [junit4]   1>                 to be flushed to stable storage, but does not cause a new
   [junit4]   1>                 searcher to be opened to make those changes visible.
   [junit4]   1>      
   [junit4]   1>               If the updateLog is enabled, then it's highly recommended to
   [junit4]   1>               have some sort of hard autoCommit to limit the log size.
   [junit4]   1>            -->
   [junit4]   1>          <autoCommit>
   [junit4]   1>            <maxTime>${solr.autoCommit.maxTime:15000}</maxTime>
   [junit4]   1>            <openSearcher>false</openSearcher>
   [junit4]   1>          </autoCommit>
   [junit4]   1>      
   [junit4]   1>          <!-- softAutoCommit is like autoCommit except it causes a
   [junit4]   1>               'soft' commit which only ensures that changes are visible
   [junit4]   1>               but does not ensure that data is synced to disk.  This is
   [junit4]   1>               faster and more near-realtime friendly than a hard commit.
   [junit4]   1>            -->
   [junit4]   1>      
   [junit4]   1>          <autoSoftCommit>
   [junit4]   1>            <maxTime>${solr.autoSoftCommit.maxTime:-1}</maxTime>
   [junit4]   1>          </autoSoftCommit>
   [junit4]   1>      
   [junit4]   1>          <!-- Update Related Event Listeners
   [junit4]   1>      
   [junit4]   1>               Various IndexWriter related events can trigger Listeners to
   [junit4]   1>               take actions.
   [junit4]   1>      
   [junit4]   1>               postCommit - fired after every commit or optimize command
   [junit4]   1>               postOptimize - fired after every optimize command
   [junit4]   1>            -->
   [junit4]   1>          <!-- The RunExecutableListener executes an external command from a
   [junit4]   1>               hook such as postCommit or postOptimize.
   [junit4]   1>      
   [junit4]   1>               exe - the name of the executable to run
   [junit4]   1>               dir - dir to use as the current working directory. (default=".")
   [junit4]   1>               wait - the calling thread waits until the executable returns.
   [junit4]   1>                      (default="true")
   [junit4]   1>               args - the arguments to pass to the program.  (default is none)
   [junit4]   1>               env - environment variables to set.  (default is none)
   [junit4]   1>            -->
   [junit4]   1>          <!-- This example shows how RunExecutableListener could be used
   [junit4]   1>               with the script based replication...
   [junit4]   1>               http://wiki.apache.org/solr/CollectionDistribution
   [junit4]   1>            -->
   [junit4]   1>          <!--
   [junit4]   1>             <listener event="postCommit" class="solr.RunExecutableListener">
   [junit4]   1>               <str name="exe">solr/bin/snapshooter</str>
   [junit4]   1>               <str name="dir">.</str>
   [junit4]   1>               <bool name="wait">true</bool>
   [junit4]   1>               <arr name="args"> <str>arg1</str> <str>arg2</str> </arr>
   [junit4]   1>               <arr name="env"> <str>MYVAR=val1</str> </arr>
   [junit4]   1>             </listener>
   [junit4]   1>            -->
   [junit4]   1>      
   [junit4]   1>        </updateHandler>
   [junit4]   1>      
   [junit4]   1>        <!-- IndexReaderFactory
   [junit4]   1>      
   [junit4]   1>             Use the following format to specify a custom IndexReaderFactory,
   [junit4]   1>             which allows for alternate IndexReader implementations.
   [junit4]   1>      
   [junit4]   1>             ** Experimental Feature **
   [junit4]   1>      
   [junit4]   1>             Please note - Using a custom IndexReaderFactory may prevent
   [junit4]   1>             certain other features from working. The API to
   [junit4]   1>             IndexReaderFactory may change without warning or may even be
   [junit4]   1>             removed from future releases if the problems cannot be
   [junit4]   1>             resolved.
   [junit4]   1>      
   [junit4]   1>      
   [junit4]   1>             ** Features that may not work with custom IndexReaderFactory **
   [junit4]   1>      
   [junit4]   1>             The ReplicationHandler assumes a disk-resident index. Using a
   [junit4]   1>             custom IndexReader implementation may cause incompatibility
   [junit4]   1>             with ReplicationHandler and may cause replication to not work
   [junit4]   1>             correctly. See SOLR-1366 for details.
   [junit4]   1>      
   [junit4]   1>          -->
   [junit4]   1>        <!--
   [junit4]   1>        <indexReaderFactory name="IndexReaderFactory" class="package.class">
   [junit4]   1>          <str name="someArg">Some Value</str>
   [junit4]   1>        </indexReaderFactory >
   [junit4]   1>        -->
   [junit4]   1>      
   [junit4]   1>        <!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   [junit4]   1>             Query section - these settings control query time things like caches
   [junit4]   1>             ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->
   [junit4]   1>        <query>
   [junit4]   1>      
   [junit4]   1>          <!-- Maximum number of clauses in each BooleanQuery,  an exception
   [junit4]   1>               is thrown if exceeded.  It is safe to increase or remove this setting,
   [junit4]   1>               since it is purely an arbitrary limit to try and catch user errors where
   [junit4]   1>               large boolean queries may not be the best implementation choice.
   [junit4]   1>            -->
   [junit4]   1>          <maxBooleanClauses>1024</maxBooleanClauses>
   [junit4]   1>      
   [junit4]   1>          <!-- Solr Internal Query Caches
   [junit4]   1>      
   [junit4]   1>               There are two implementations of cache available for Solr,
   [junit4]   1>               LRUCache, based on a synchronized LinkedHashMap, and
   [junit4]   1>               FastLRUCache, based on a ConcurrentHashMap.
   [junit4]   1>      
   [junit4]   1>               FastLRUCache has faster gets and slower puts in single
   [junit4]   1>               threaded operation and thus is generally faster than LRUCache
   [junit4]   1>               when the hit ratio of the cache is high (> 75%), and may be
   [junit4]   1>               faster under other scenarios on multi-cpu systems.
   [junit4]   1>          -->
   [junit4]   1>      
   [junit4]   1>          <!-- Filter Cache
   [junit4]   1>      
   [junit4]   1>               Cache used by SolrIndexSearcher for filters (DocSets),
   [junit4]   1>               unordered sets of *all* documents that match a query.  When a
   [junit4]   1>               new searcher is opened, its caches may be prepopulated or
   [junit4]   1>               "autowarmed" using data from caches in the old searcher.
   [junit4]   1>               autowarmCount is the number of items to prepopulate.  For
   [junit4]   1>               LRUCache, the autowarmed items will be the most recently
   [junit4]   1>               accessed items.
   [junit4]   1>      
   [junit4]   1>               Parameters:
   [junit4]   1>                 class - the SolrCache implementation LRUCache or
   [junit4]   1>                     (LRUCache or FastLRUCache)
   [junit4]   1>                 size - the maximum number of entries in the cache
   [junit4]   1>                 initialSize - the initial capacity (number of entries) of
   [junit4]   1>                     the cache.  (see java.util.HashMap)
   [junit4]   1>                 autowarmCount - the number of entries to prepopulate from
   [junit4]   1>                     and old cache.
   [junit4]   1>                 maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
   [junit4]   1>                            to occupy. Note that when this option is specified, the size
   [junit4]   1>                            and initialSize parameters are ignored.
   [junit4]   1>            -->
   [junit4]   1>          <filterCache class="solr.FastLRUCache"
   [junit4]   1>                       size="512"
   [junit4]   1>                       initialSize="512"
   [junit4]   1>                       autowarmCount="0"/>
   [junit4]   1>      
   [junit4]   1>          <!-- Query Result Cache
   [junit4]   1>      
   [junit4]   1>               Caches results of searches - ordered lists of document ids
   [junit4]   1>               (DocList) based on a query, a sort, and the range of documents requested.
   [junit4]   1>               Additional supported parameter by LRUCache:
   [junit4]   1>                  maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
   [junit4]   1>                             to occupy
   [junit4]   1>            -->
   [junit4]   1>          <queryResultCache class="solr.LRUCache"
   [junit4]   1>                            size="512"
   [junit4]   1>                            initialSize="512"
   [junit4]   1>                            autowarmCount="0"/>
   [junit4]   1>      
   [junit4]   1>          <!-- Document Cache
   [junit4]   1>      
   [junit4]   1>               Caches Lucene Document objects (the stored fields for each
   [junit4]   1>               document).  Since Lucene internal document ids are transient,
   [junit4]   1>               this cache will not be autowarmed.
   [junit4]   1>            -->
   [junit4]   1>          <documentCache class="solr.LRUCache"
   [junit4]   1>                         size="512"
   [junit4]   1>                         initialSize="512"
   [junit4]   1>                         autowarmCount="0"/>
   [junit4]   1>      
   [junit4]   1>          <!-- custom cache currently used by block join -->
   [junit4]   1>          <cache name="perSegFilter"
   [junit4]   1>                 class="solr.search.LRUCache"
   [junit4]   1>                 size="10"
   [junit4]   1>                 initialSize="0"
   [junit4]   1>                 autowarmCount="10"
   [junit4]   1>                 regenerator="solr.NoOpRegenerator" />
   [junit4]   1>      
   [junit4]   1>          <!-- Field Value Cache
   [junit4]   1>      
   [junit4]   1>               Cache used to hold field values that are quickly accessible
   [junit4]   1>               by document id.  The fieldValueCache is created by default
   [junit4]   1>               even if not configured here.
   [junit4]   1>            -->
   [junit4]   1>          <!--
   [junit4]   1>             <fieldValueCache class="solr.FastLRUCache"
   [junit4]   1>                              size="512"
   [junit4]   1>                              autowarmCount="128"
   [junit4]   1>                              showItems="32" />
   [junit4]   1>            -->
   [junit4]   1>      
   [junit4]   1>          <!-- Custom Cache
   [junit4]   1>      
   [junit4]   1>               Example of a generic cache.  These caches may be accessed by
   [junit4]   1>               name through SolrIndexSearcher.getCache(),cacheLookup(), and
   [junit4]   1>               cacheInsert().  The purpose is to enable easy caching of
   [junit4]   1>               user/application level data.  The regenerator argument should
   [junit4]   1>               be specified as an implementation of solr.CacheRegenerator
   [junit4]   1>               if autowarming is desired.
   [junit4]   1>            -->
   [junit4]   1>          <!--
   [junit4]   1>             <cache name="myUserCache"
   [junit4]   1>                    class="solr.LRUCache"
   [junit4]   1>                    size="4096"
   [junit4]   1>                    initialSize="1024"
   [junit4]   1>                    autowarmCount="1024"
   [junit4]   1>                    regenerator="com.mycompany.MyRegenerator"
   [junit4]   1>                    />
   [junit4]   1>            -->
   [junit4]   1>      
   [junit4]   1>      
   [junit4]   1>          <!-- Lazy Field Loading
   [junit4]   1>      
   [junit4]   1>               If true, stored fields that are not requested will be loaded
   [junit4]   1>               lazily.  This can result in a significant speed improvement
   [junit4]   1>               if the usual case is to not load all stored fields,
   [junit4]   1>               especially if the skipped fields are large compressed text
   [junit4]   1>               fields.
   [junit4]   1>          -->
   [junit4]   1>          <enableLazyFieldLoading>true</enableLazyFieldLoading>
   [junit4]   1>      
   [junit4]   1>          <!-- Use Filter For Sorted Query
   [junit4]   1>      
   [junit4]   1>               A possible optimization that attempts to use a filter to
   [junit4]   1>               satisfy a search.  If the requested sort does not include
   [junit4]   1>               score, then the filterCache will be checked for a filter
   [junit4]   1>               matching the query. If found, the filter will be used as the
   [junit4]   1>               source of document ids, and then the sort will be applied to
   [junit4]   1>               that.
   [junit4]   1>      
   [junit4]   1>               For most situations, this will not be useful unless you
   [junit4]   1>               frequently get the same search repeatedly with different sort
   [junit4]   1>               options, and none of them ever use "score"
   [junit4]   1>            -->
   [junit4]   1>          <!--
   [junit4]   1>             <useFilterForSortedQuery>true</useFilterForSortedQuery>
   [junit4]   1>            -->
   [junit4]   1>      
   [junit4]   1>          <!-- Result Window Size
   [junit4]   1>      
   [junit4]   1>               An optimization for use with the queryResultCache.  When a search
   [junit4]   1>               is requested, a superset of the requested number of document ids
   [junit4]   1>               are collected.  For example, if a search for a particular query
   [junit4]   1>               requests matching documents 10 through 19, and queryWindowSize is 50,
   [junit4]   1>               then documents 0 through 49 will be collected and cached.  Any further
   [junit4]   1>               requests in that range can be satisfied via the cache.
   [junit4]   1>            -->
   [junit4]   1>          <queryResultWindowSize>20</queryResultWindowSize>
   [junit4]   1>      
   [junit4]   1>          <!-- Maximum number of documents to cache for any entry in the
   [junit4]   1>               queryResultCache.
   [junit4]   1>            -->
   [junit4]   1>          <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
   [junit4]   1>      
   [junit4]   1>          <!-- Query Related Event Listeners
   [junit4]   1>      
   [junit4]   1>               Various IndexSearcher related events can trigger Listeners to
   [junit4]   1>               take actions.
   [junit4]   1>      
   [junit4]   1>               newSearcher - fired whenever a new searcher is being prepared
   [junit4]   1>               and there is a current searcher handling requests (aka
   [junit4]   1>               registered).  It can be used to prime certain caches to
   [junit4]   1>               prevent long request times for certain requests.
   [junit4]   1>      
   [junit4]   1>               firstSearcher - fired whenever a new searcher is being
   [junit4]   1>               prepared but there is no current registered searcher to handle
   [junit4]   1>               requests or to gain autowarming data from.
   [junit4]   1>      
   [junit4]   1>      
   [junit4]   1>            -->
   [junit4]   1>          <!-- QuerySenderListener takes an array of NamedList and executes a
   [junit4]   1>               local query request for each NamedList in sequence.
   [junit4]   1>            -->
   [junit4]   1>          <listener event="newSearcher" class="solr.QuerySenderListener">
   [junit4]   1>            <arr name="queries">
   [junit4]   1>              <!--
   [junit4]   1>                 <lst><str name="q">solr</str><str name="sort">price asc</str></lst>
   [junit4]   1>                 <lst><str name="q">rocks</str><str name="sort">weight asc</str></lst>
   [junit4]   1>                -->
   [junit4]   1>            </arr>
   [junit4]   1>          </listener>
   [junit4]   1>          <listener event="firstSearcher" class="solr.QuerySenderListener">
   [junit4]   1>            <arr name="queries">
   [junit4]   1>              <!--
   [junit4]   1>              <lst>
   [junit4]   1>                <str name="q">static firstSearcher warming in solrconfig.xml</str>
   [junit4]   1>              </lst>
   [junit4]   1>              -->
   [junit4]   1>            </arr>
   [junit4]   1>          </listener>
   [junit4]   1>      
   [junit4]   1>          <!-- Use Cold Searcher
   [junit4]   1>      
   [junit4]   1>               If a search request comes in and there is no current
   [junit4]   1>               registered searcher, then immediately register the still
   [junit4]   1>               warming searcher and use it.  If "false" then all requests
   [junit4]   1>               will block until the first searcher is done warming.
   [junit4]   1>            -->
   [junit4]   1>          <useColdSearcher>false</useColdSearcher>
   [junit4]   1>      
   [junit4]   1>        </query>
   [junit4]   1>      
   [junit4]   1>      
   [junit4]   1>        <!-- Request Dispatcher
   [junit4]   1>      
   [junit4]   1>             This section contains instructions for how the SolrDispatchFilter
   [junit4]   1>             should behave when processing requests for this SolrCore.
   [junit4]   1>      
   [junit4]   1>          -->
   [junit4]   1>        <requestDispatcher>
   [junit4]   1>          <!-- Request Parsing
   [junit4]   1>      
   [junit4]   1>               These settings indicate how Solr Requests may be parsed, and
   [junit4]   1>               what restrictions may be placed on the ContentStreams from
   [junit4]   1>               those requests
   [junit4]   1>      
   [junit4]   1>               enableRemoteStreaming - enables use of the stream.file
   [junit4]   1>               and stream.url parameters for specifying remote streams.
   [junit4]   1>      
   [junit4]   1>               multipartUploadLimitInKB - specifies the max size (in KiB) of
   [junit4]   1>               Multipart File Uploads that Solr will allow in a Request.
   [junit4]   1>      
   [junit4]   1>               formdataUploadLimitInKB - specifies the max size (in KiB) of
   [junit4]   1>               form data (application/x-www-form-urlencoded) sent via
   [junit4]   1>               POST. You can use POST to pass request parameters not
   [junit4]   1>               fitting into the URL.
   [junit4]   1>      
   [junit4]   1>               addHttpRequestToContext - if set to true, it will instruct
   [junit4]   1>               the requestParsers to include the original HttpServletRequest
   [junit4]   1>               object in the context map of the SolrQueryRequest under the
   [junit4]   1>               key "httpRequest". It will not be used by any of the existing
   [junit4]   1>               Solr components, but may be useful when developing custom
   [junit4]   1>               plugins.
   [junit4]   1>      
   [junit4]   1>               *** WARNING ***
   [junit4]   1>               Before enabling remote streaming, you should make sure your
   [junit4]   1>               system has authentication enabled.
   [junit4]   1>      
   [junit4]   1>          <requestParsers enableRemoteStreaming="false"
   [junit4]   1>                          multipartUploadLimitInKB="-1"
   [junit4]   1>                          formdataUploadLimitInKB="-1"
   [junit4]   1>                          addHttpRequestToContext="false"/>
   [junit4]   1>            -->
   [junit4]   1>      
   [junit4]   1>          <!-- HTTP Caching
   [junit4]   1>      
   [junit4]   1>               Set HTTP caching related parameters (for proxy caches and clients).
   [junit4]   1>      
   [junit4]   1>               The options below instruct Solr not to output any HTTP Caching
   [junit4]   1>               related headers
   [junit4]   1>            -->
   [junit4]   1>          <httpCaching never304="true" />
   [junit4]   1>          <!-- If you include a <cacheControl> directive, it will be used to
   [junit4]   1>               generate a Cache-Control header (as well as an Expires header
   [junit4]   1>               if the value contains "max-age=")
   [junit4]   1>      
   [junit4]   1>               By default, no Cache-Control header is generated.
   [junit4]   1>      
   [junit4]   1>               You can use the <cacheControl> option even if you have set
   [junit4]   1>               never304="true"
   [junit4]   1>            -->
   [junit4]   1>          <!--
   [junit4]   1>             <httpCaching never304="true" >
   [junit4]   1>               <cacheControl>max-age=30, public</cacheControl>
   [junit4]   1>             </httpCaching>
   [junit4]   1>            -->
   [junit4]   1>          <!-- To enable Solr to respond with automatically generated HTTP
   [junit4]   1>               Caching headers, and to response to Cache Validation requests
   [junit4]   1>               correctly, set the value of never304="false"
   [junit4]   1>      
   [junit4]   1>               This will cause Solr to generate Last-Modified and ETag
   [junit4]   1>               headers based on the properties of the Index.
   [junit4]   1>      
   [junit4]   1>               The following options can also be specified to affect the
   [junit4]   1>               values of these headers...
   [junit4]   1>      
   [junit4]   1>               lastModFrom - the default value is "openTime" which means the
   [junit4]   1>               Last-Modified value (and validation against If-Modified-Since
   [junit4]   1>               requests) will all be relative to when the current Searcher
   [junit4]   1>               was opened.  You can change it to lastModFrom="dirLastMod" if
   [junit4]   1>               you want the value to exactly correspond to when the physical
   [junit4]   1>               index was last modified.
   [junit4]   1>      
   [junit4]   1>               etagSeed="..." is an option you can change to force the ETag
   [junit4]   1>               header (and validation against If-None-Match requests) to be
   [junit4]   1>               different even if the index has not changed (ie: when making
   [junit4]   1>               significant changes to your config file)
   [junit4]   1>      
   [junit4]   1>               (lastModifiedFrom and etagSeed are both ignored if you use
   [junit4]   1>               the never304="true" option)
   [junit4]   1>            -->
   [junit4]   1>          <!--
   [junit4]   1>             <httpCaching lastModifiedFrom="openTime"
   [junit4]   1>                          etagSeed="Solr">
   [junit4]   1>               <cacheControl>max-age=30, public</cacheControl>
   [junit4]   1>             </httpCaching>
   [junit4]   1>            -->
   [junit4]   1>        </requestDispatcher>
   [junit4]   1>      
   [junit4]   1>        <!-- Request Handlers
   [junit4]   1>      
   [junit4]   1>             http://wiki.apache.org/solr/SolrRequestHandler
   [junit4]   1>      
   [junit4]   1>             Incoming queries will be dispatched to a specific handler by name
   [junit4]   1>             based on the path specified in the request.
   [junit4]   1>      
   [junit4]   1>             If a Request Handler is declared with startup="lazy", then it will
   [junit4]   1>             not be initialized until the first request that uses it.
   [junit4]   1>      
   [junit4]   1>          -->
   [junit4]   1>        <!-- SearchHandler
   [junit4]   1>      
   [junit4]   1>             http://wiki.apache.org/solr/SearchHandler
   [junit4]   1>      
   [junit4]   1>             For processing Search Queries, the primary Request Handler
   [junit4]   1>             provided with Solr is "SearchHandler" It delegates to a sequent
   [junit4]   1>             of SearchComponents (see below) and supports distributed
   [junit4]   1>             queries across multiple shards
   [junit4]   1>          -->
   [junit4]   1>        <requestHandler name="/select" class="solr.SearchHandler">
   [junit4]   1>          <!-- default values for query parameters can be specified, these
   [junit4]   1>               will be overridden by parameters in the request
   [junit4]   1>            -->
   [junit4]   1>          <lst name="defaults">
   [junit4]   1>            <str name="echoParams">explicit</str>
   [junit4]   1>            <int name="rows">10</int>
   [junit4]   1>            <!-- Default search field
   [junit4]   1>               <str name="df">text</str> 
   [junit4]   1>              -->
   [junit4]   1>            <!-- Change from JSON to XML format (the default prior to Solr 7.0)
   [junit4]   1>               <str name="wt">xml</str> 
   [junit4]   1>              -->
   [junit4]   1>          </lst>
   [junit4]   1>          <!-- In addition to defaults, "appends" params can be specified
   [junit4]   1>               to identify values which should be appended to the list of
   [junit4]   1>               multi-val params from the query (or the existing "defaults").
   [junit4]   1>            -->
   [junit4]   1>          <!-- In this example, the param "fq=instock:true" would be appended to
   [junit4]   1>               any query time fq params the user may specify, as a mechanism for
   [junit4]   1>               partitioning the index, independent of any user selected filtering
   [junit4]   1>               that may also be desired (perhaps as a result of faceted searching).
   [junit4]   1>      
   [junit4]   1>               NOTE: there is *absolutely* nothing a client can do to prevent these
   [junit4]   1>               "appends" values from being used, so don't use this mechanism
   [junit4]   1>               unless you are sure you always want it.
   [junit4]   1>            -->
   [junit4]   1>          <!--
   [junit4]   1>             <lst name="appends">
   [junit4]   1>               <str name="fq">inStock:true</str>
   [junit4]   1>             </lst>
   [junit4]   1>            -->
   [junit4]   1>          <!-- "invariants" are a way of letting the Solr maintainer lock down
   [junit4]   1>               the options available to Solr clients.  Any params values
   [junit4]   1>               specified here are used regardless of what values may be specified
   [junit4]   1>               in either the query, the "defaults", or the "appends" params.
   [junit4]   1>      
   [junit4]   1>               In this example, the facet.field and facet.query params would
   [junit4]   1>               be fixed, limiting the facets clients can use.  Faceting is
   [junit4]   1>               not turned on by default - but if the client does specify
   [junit4]   1>               facet=true in the request, these are the only facets they
   [junit4]   1>               will be able to see counts for; regardless of what other
   [junit4]   1>               facet.field or facet.query params they may specify.
   [junit4]   1>      
   [junit4]   1>               NOTE: there is *absolutely* nothing a client can do to prevent these
   [junit4]   1>               "invariants" values from being used, so don't use this mechanism
   [junit4]   1>               unless you are sure you always want it.
   [junit4]   1>            -->
   [junit4]   1>          <!--
   [junit4]   1>             <lst name="invariants">
   [junit4]   1>               <str name="facet.field">cat</str>
   [junit4]   1>               <str name="facet.field">manu_exact</str>
   [junit4]   1>               <str name="facet.query">price:[* TO 500]</str>
   [junit4]   1>               <str name="facet.query">price:[500 TO *]</str>
   [junit4]   1>             </lst>
   [junit4]   1>            -->
   [junit4]   1>          <!-- If the default list of SearchComponents is not desired, that
   [junit4]   1>               list can either be overridden completely, or components can be
   [junit4]   1>               prepended or appended to the default list.  (see below)
   [junit4]   1>            -->
   [junit4]   1>          <!--
   [junit4]   1>             <arr name="components">
   [junit4]   1>               <str>nameOfCustomComponent1</str>
   [junit4]   1>               <str>nameOfCustomComponent2</str>
   [junit4]   1>             </arr>
   [junit4]   1>            -->
   [junit4]   1>        </requestHandler>
   [junit4]   1>      
   [junit4]   1>        <!-- A request handler that returns indented JSON by default -->
   [junit4]   1>        <requestHandler name="/query" class="solr.SearchHandler">
   [junit4]   1>          <lst name="defaults">
   [junit4]   1>            <str name="echoParams">explicit</str>
   [junit4]   1>            <str name="wt">json</str>
   [junit4]   1>            <str name="indent">true</str>
   [junit4]   1>          </lst>
   [junit4]   1>        </requestHandler>
   [junit4]   1>      
   [junit4]   1>      
   [junit4]   1>        <!-- A Robust Example
   [junit4]   1>      
   [junit4]   1>             This example SearchHandler declaration shows off usage of the
   [junit4]   1>             SearchHandler with many defaults declared
   [junit4]   1>      
   [junit4]   1>             Note that multiple instances of the same Request Handler
   [junit4]   1>             (SearchHandler) can be registered multiple times with different
   [junit4]   1>             names (and different init parameters)
   [junit4]   1>          -->
   [junit4]   1>        <requestHandler name="/browse" class="solr.SearchHandler" useParams="query,facets,velocity,browse">
   [junit4]   1>          <lst name="defaults">
   [junit4]   1>            <str name="echoParams">explicit</str>
   [junit4]   1>          </lst>
   [junit4]   1>        </requestHandler>
   [junit4]   1>      
   [junit4]   1>        <initParams path="/update/**,/query,/select,/tvrh,/elevate,/spell,/browse">
   [junit4]   1>          <lst name="defaults">
   [junit4]   1>            <str name="df">_text_</str>
   [junit4]   1>          </lst>
   [junit4]   1>        </initParams>
   [junit4]   1>      
   [junit4]   1>        <!-- Solr Cell Update Request Handler
   [junit4]   1>      
   [junit4]   1>             http://wiki.apache.org/solr/ExtractingRequestHandler
   [junit4]   1>      
   [junit4]   1>          -->
   [junit4]   1>        <requestHandler name="/update/extract"
   [junit4]   1>                        startup="lazy"
   [junit4]   1>                        class="solr.extraction.ExtractingRequestHandler" >
   [junit4]   1>          <lst name="defaults">
   [junit4]   1>            <str name="lowernames">true</str>
   [junit4]   1>            <str name="fmap.meta">ignored_</str>
   [junit4]   1>            <str name="fmap.content">_text_</str>
   [junit4]   1>          </lst>
   [junit4]   1>        </requestHandler>
   [junit4]   1>      
   [junit4]   1>        <!-- Search Components
   [junit4]   1>      
   [junit4]   1>             Search components are registered to SolrCore and used by
   [junit4]   1>             instances of SearchHandler (which can access them by name)
   [junit4]   1>      
   [junit4]   1>             By default, the following components are available:
   [junit4]   1>      
   [junit4]   1>             <searchComponent name="query"     class="solr.QueryComponent" />
   [junit4]   1>             <searchComponent name="facet"     class="solr.FacetComponent" />
   [junit4]   1>             <searchComponent name="mlt"       class="solr.MoreLikeThisComponent" />
   [junit4]   1>             <searchComponent name="highlight" class="solr.HighlightComponent" />
   [junit4]   1>             <searchComponent name="stats"     class="solr.StatsComponent" />
   [junit4]   1>             <searchComponent name="debug"     class="solr.DebugComponent" />
   [junit4]   1>      
   [junit4]   1>             Default configuration in a requestHandler would look like:
   [junit4]   1>      
   [junit4]   1>             <arr name="components">
   [junit4]   1>               <str>query</str>
   [junit4]   1>               <str>facet</str>
   [junit4]   1>               <str>mlt</str>
   [junit4]   1>               <str>highlight</str>
   [junit4]   1>               <str>stats</str>
   [junit4]   1>               <str>debug</str>
   [junit4]   1>             </arr>
   [junit4]   1>      
   [junit4]   1>             If you register a searchComponent to one of the standard names,
   [junit4]   1>             that will be used instead of the default.
   [junit4]   1>      
   [junit4]   1>             To insert components before or after the 'standard' components, use:
   [junit4]   1>      
   [junit4]   1>             <arr name="first-components">
   [junit4]   1>               <str>myFirstComponentName</str>
   [junit4]   1>             </arr>
   [junit4]   1>      
   [junit4]   1>             <arr name="last-components">
   [junit4]   1>               <str>myLastComponentName</str>
   [junit4]   1>             </arr>
   [junit4]   1>      
   [junit4]   1>             NOTE: The component registered with the name "debug" will
   [junit4]   1>             always be executed after the "last-components"
   [junit4]   1>      
   [junit4]   1>           -->
   [junit4]   1>      
   [junit4]   1>        <!-- Spell Check
   [junit4]   1>      
   [junit4]   1>             The spell check component can return a list of alternative spelling
   [junit4]   1>             suggestions.
   [junit4]   1>      
   [junit4]   1>             http://wiki.apache.org/solr/SpellCheckComponent
   [junit4]   1>          -->
   [junit4]   1>        <searchComponent name="spellcheck" class="solr.SpellCheckComponent">
   [junit4]   1>      
   [junit4]   1>          <str name="queryAnalyzerFieldType">text_general</str>
   [junit4]   1>      
   [junit4]   1>          <!-- Multiple "Spell Checkers" can be declared and used by this
   [junit4]   1>               component
   [junit4]   1>            -->
   [junit4]   1>      
   [junit4]   1>          <!-- a spellchecker built from a field of the main index -->
   [junit4]   1>          <lst name="spellchecker">
   [junit4]   1>            <str name="name">default</str>
   [junit4]   1>            <str name="field">_text_</str>
   [junit4]   1>            <str name="classname">solr.DirectSolrSpellChecker</str>
   [junit4]   1>            <!-- the spellcheck distance measure used, the default is the internal levenshtein -->
   [junit4]   1>            <str name="distanceMeasure">internal</str>
   [junit4]   1>            <!-- minimum accuracy needed to be considered a valid spellcheck suggestion -->
   [junit4]   1>            <float name="accuracy">0.5</float>
   [junit4]   1>            <!-- the maximum #edits we consider when enumerating terms: can be 1 or 2 -->
   [junit4]   1>            <int name="maxEdits">2</int>
   [junit4]   1>            <!-- the minimum shared prefix when enumerating terms -->
   [junit4]   1>            <int name="minPrefix">1</int>
   [junit4]   1>            <!-- maximum number of inspections per result. -->
   [junit4]   1>            <int name="maxInspections">5</int>
   [junit4]   1>            <!-- minimum length of a query term to be considered for correction -->
   [junit4]   1>            <int name="minQueryLength">4</int>
   [junit4]   1>            <!-- maximum threshold of documents a query term can appear to be considered for correction -->
   [junit4]   1>            <float name="maxQueryFrequency">0.01</float>
   [junit4]   1>            <!-- uncomment this to require suggestions to occur in 1% of the documents
   [junit4]   1>              <float name="thresholdTokenFrequency">.01</float>
   [junit4]   1>            -->
   [junit4]   1>          </lst>
   [junit4]   1>      
   [junit4]   1>          <!-- a spellchecker that can break or combine words.  See "/spell" handler below for usage -->
   [junit4]   1>          <!--
   [junit4]   1>          <lst name="spellchecker">
   [junit4]   1>            <str name="name">wordbreak</str>
   [junit4]   1>            <str name="classname">solr.WordBreakSolrSpellChecker</str>
   [junit4]   1>            <str name="field">name</str>
   [junit4]   1>            <str name="combineWords">true</str>
   [junit4]   1>            <str name="breakWords">true</str>
   [junit4]   1>            <int name="maxChanges">10</int>
   [junit4]   1>          </lst>
   [junit4]   1>          -->
   [junit4]   1>        </searchComponent>
   [junit4]   1>      
   [junit4]   1>        <!-- A request handler for demonstrating the spellcheck component.
   [junit4]   1>      
   [junit4]   1>             NOTE: This is purely as an example.  The whole purpose of the
   [junit4]   1>             SpellCheckComponent is to hook it into the request handler that
   [junit4]   1>             handles your normal user queries so that a separate request is
   [junit4]   1>             not needed to get suggestions.
   [junit4]   1>      
   [junit4]   1>             IN OTHER WORDS, THERE IS REALLY GOOD CHANCE THE SETUP BELOW IS
   [junit4]   1>             NOT WHAT YOU WANT FOR YOUR PRODUCTION SYSTEM!
   [junit4]   1>      
   [junit4]   1>             See http://wiki.apache.org/solr/SpellCheckComponent for details
   [junit4]   1>             on the request parameters.
   [junit4]   1>          -->
   [junit4]   1>        <requestHandler name="/spell" class="solr.SearchHandler" startup="lazy">
   [junit4]   1>          <lst name="defaults">
   [junit4]   1>            <!-- Solr will use suggestions from both the 'default' spellchecker
   [junit4]   1>                 and from the 'wordbreak' spellchecker and combine them.
   [junit4]   1>                 collations (re-written queries) can include a combination of
   [junit4]   1>                 corrections from both spellcheckers -->
   [junit4]   1>            <str name="spellcheck.dictionary">default</str>
   [junit4]   1>            <str name="spellcheck">on</str>
   [junit4]   1>            <str name="spellcheck.extendedResults">true</str>
   [junit4]   1>            <str name="spellcheck.count">10</str>
   [junit4]   1>            <str name="spellcheck.alternativeTermCount">5</str>
   [junit4]   1>            <str name="spellcheck.maxResultsForSuggest">5</str>
   [junit4]   1>            <str name="spellcheck.collate">true</str>
   [junit4]   1>            <str name="spellcheck.collateExtendedResults">true</str>
   [junit4]   1>            <str name="spellcheck.maxCollationTries">10</str>
   [junit4]   1>            <str name="spellcheck.maxCollations">5</str>
   [junit4]   1>          </lst>
   [junit4]   1>          <arr name="last-components">
   [junit4]   1>            <str>spellcheck</str>
   [junit4]   1>          </arr>
   [junit4]   1>        </requestHandler>
   [junit4]   1>      
   [junit4]   1>        <!-- Term Vector Component
   [junit4]   1>      
   [junit4]   1>             http://wiki.apache.org/solr/TermVectorComponent
   [junit4]   1>          -->
   [junit4]   1>        <searchComponent name="tvComponent" class="solr.TermVectorComponent"/>
   [junit4]   1>      
   [junit4]   1>        <!-- A request handler for demonstrating the term vector component
   [junit4]   1>      
   [junit4]   1>             This is purely as an example.
   [junit4]   1>      
   [junit4]   1>             In reality you will likely want to add the component to your
   [junit4]   1>             already specified request handlers.
   [junit4]   1>          -->
   [junit4]   1>        <requestHandler name="/tvrh" class="solr.SearchHandler" startup="lazy">
   [junit4]   1>          <lst name="defaults">
   [junit4]   1>            <bool name="tv">true</bool>
   [junit4]   1>          </lst>
   [junit4]   1>          <arr name="last-components">
   [junit4]   1>            <str>tvComponent</str>
   [junit4]   1>          </arr>
   [junit4]   1>        </requestHandler>
   [junit4]   1>      
   [junit4]   1>        <!-- Clustering Component. (Omitted here. See the default Solr example for a typical configuration.) -->
   [junit4]   1>      
   [junit4]   1>        <!-- Terms Component
   [junit4]   1>      
   [junit4]   1>             http://wiki.apache.org/solr/TermsComponent
   [junit4]   1>      
   [junit4]   1>             A component to return terms and document frequency of those
   [junit4]   1>             terms
   [junit4]   1>          -->
   [junit4]   1>        <searchComponent name="terms" class="solr.TermsComponent"/>
   [junit4]   1>      
   [junit4]   1>        <!-- A request handler for demonstrating the terms component -->
   [junit4]   1>        <requestHandler name="/terms" class="solr.SearchHandler" startup="lazy">
   [junit4]   1>          <lst name="defaults">
   [junit4]   1>            <bool name="terms">true</bool>
   [junit4]   1>            <bool name="distrib">false</bool>
   [junit4]   1>          </lst>
   [junit4]   1>          <arr name="components">
   [junit4]   1>            <str>terms</str>
   [junit4]   1>          </arr>
   [junit4]   1>        </requestHandler>
   [junit4]   1>      
   [junit4]   1>      
   [junit4]   1>        <!-- Query Elevation Component
   [junit4]   1>      
   [junit4]   1>             http://wiki.apache.org/solr/QueryElevationComponent
   [junit4]   1>      
   [junit4]   1>             a search component that enables you to configure the top
   [junit4]   1>             results for a given query regardless of the normal lucene
   [junit4]   1>             scoring.
   [junit4]   1>          -->
   [junit4]   1>        <searchComponent name="elevator" class="solr.QueryElevationComponent" >
   [junit4]   1>          <!-- pick a fieldType to analyze queries -->
   [junit4]   1>          <str name="queryFieldType">string</str>
   [junit4]   1>        </searchComponent>
   [junit4]   1>      
   [junit4]   1>        <!-- A request handler for demonstrating the elevator component -->
   [junit4]   1>        <requestHandler name="/elevate" class="solr.SearchHandler" startup="lazy">
   [junit4]   1>          <lst name="defaults">
   [junit4]   1>            <str name="echoParams">explicit</str>
   [junit4]   1>          </lst>
   [junit4]   1>          <arr name="last-components">
   [junit4]   1>            <str>elevator</str>
   [junit4]   1>          </arr>
   [junit4]   1>        </requestHandler>
   [junit4]   1>      
   [junit4]   1>        <!-- Highlighting Component
   [junit4]   1>      
   [junit4]   1>             http://wiki.apache.org/solr/HighlightingParameters
   [junit4]   1>          -->
   [junit4]   1>        <searchComponent class="solr.HighlightComponent" name="highlight">
   [junit4]   1>          <highlighting>
   [junit4]   1>            <!-- Configure the standard fragmenter -->
   [junit4]   1>            <!-- This could most likely be commented out in the "default" case -->
   [junit4]   1>            <fragmenter name="gap"
   [junit4]   1>                        default="true"
   [junit4]   1>                        class="solr.highlight.GapFragmenter">
   [junit4]   1>              <lst name="defaults">
   [junit4]   1>                <int name="hl.fragsize">100</int>
   [junit4]   1>              </lst>
   [junit4]   1>            </fragmenter>
   [junit4]   1>      
   [junit4]   1>            <!-- A regular-expression-based fragmenter
   [junit4]   1>                 (for sentence extraction)
   [junit4]   1>              -->
   [junit4]   1>            <fragmenter name="regex"
   [junit4]   1>                        class="solr.highlight.RegexFragmenter">
   [junit4]   1>              <lst name="defaults">
   [junit4]   1>                <!-- slightly smaller fragsizes work better because of slop -->
   [junit4]   1>                <int name="hl.fragsize">70</int>
   [junit4]   1>                <!-- allow 50% slop on fragment sizes -->
   [junit4]   1>                <float name="hl.regex.slop">0.5</float>
   [junit4]   1>                <!-- a basic sentence pattern -->
   [junit4]   1>                <str name="hl.regex.pattern">[-\w ,/\n\&quot;&apos;]{20,200}</str>
   [junit4]   1>              </lst>
   [junit4]   1>            </fragmenter>
   [junit4]   1>      
   [junit4]   1>            <!-- Configure the standard formatter -->
   [junit4]   1>            <formatter name="html"
   [junit4]   1>                       default="true"
   [junit4]   1>                       class="solr.highlight.HtmlFormatter">
   [junit4]   1>              <lst name="defaults">
   [junit4]   1>                <str name="hl.simple.pre"><![CDATA[<em>]]></str>
   [junit4]   1>                <str name="hl.simple.post"><![CDATA[</em>]]></str>
   [junit4]   1>              </lst>
   [junit4]   1>            </formatter>
   [junit4]   1>      
   [junit4]   1>            <!-- Configure the standard encoder -->
   [junit4]   1>            <encoder name="html"
   [junit4]   1>                     class="solr.highlight.HtmlEncoder" />
   [junit4]   1>      
   [junit4]   1>            <!-- Configure the standard fragListBuilder -->
   [junit4]   1>            <fragListBuilder name="simple"
   [junit4]   1>                             class="solr.highlight.SimpleFragListBuilder"/>
   [junit4]   1>      
   [junit4]   1>            <!-- Configure the single fragListBuilder -->
   [junit4]   1>            <fragListBuilder name="single"
   [junit4]   1>                             class="solr.highlight.SingleFragListBuilder"/>
   [junit4]   1>      
   [junit4]   1>            <!-- Configure the

[...truncated too long message...]

r) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 3239868 INFO  (TEST-SolrSlf4jReporterTest.testReporter-seed#[1E21CB467E045E61]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testReporter
   [junit4]   2> 3246127 INFO  (TEST-SolrSlf4jReporterTest.testReporter-seed#[1E21CB467E045E61]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testReporter
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SolrSlf4jReporterTest -Dtests.method=testReporter -Dtests.seed=1E21CB467E045E61 -Dtests.slow=true -Dtests.locale=ko -Dtests.timezone=America/Panama -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 6.26s J1 | SolrSlf4jReporterTest.testReporter <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: No 'solr.node' logs in: {numFound=0,start=0,docs=[]}
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([1E21CB467E045E61:41C5E6711508CD24]:0)
   [junit4]    > 	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:85)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 3246128 INFO  (SUITE-SolrSlf4jReporterTest-seed#[1E21CB467E045E61]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.metrics.reporters.SolrSlf4jReporterTest_1E21CB467E045E61-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=146, maxMBSortInHeap=6.817432692479736, sim=RandomSimilarity(queryNorm=true): {}, locale=ko, timezone=America/Panama
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=108904296,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [TestSuggestSpellingConverter, TestCollapseQParserPlugin, DistributedQueryComponentCustomSortTest, AnalysisErrorHandlingTest, ConvertedLegacyTest, ChangedSchemaMergeTest, TestInfoStreamLogging, SolrIndexConfigTest, DocValuesNotIndexedTest, TestExportWriter, ConnectionManagerTest, SuggesterTSTTest, RollingRestartTest, TestConfigsApi, TestSimpleTrackingShardHandler, TestRecovery, PolyFieldTest, TestUniqueKeyFieldResource, MetricsConfigTest, JSONWriterTest, UpdateParamsTest, TestConfigSetsAPIExclusivity, TestFieldTypeResource, JsonLoaderTest, LoggingHandlerTest, StatsComponentTest, TestCloudInspectUtil, TestSolrCloudWithHadoopAuthPlugin, DOMUtilTest, ShardRoutingCustomTest, SpellingQueryConverterTest, TestManagedResource, TestIndexSearcher, RequestLoggingTest, BigEndianAscendingWordDeserializerTest, TestRemoteStreaming, DocValuesMultiTest, TestBM25SimilarityFactory, TestSolrIndexConfig, TestSchemaManager, TestObjectReleaseTracker, TestBadConfig, SharedFSAutoReplicaFailoverTest, LeaderInitiatedRecoveryOnCommitTest, BlockDirectoryTest, CloneFieldUpdateProcessorFactoryTest, TestAnalyzedSuggestions, CoreSorterTest, SolrPluginUtilsTest, FileUtilsTest, CollectionsAPIDistributedZkTest, TestConfig, DirectSolrSpellCheckerTest, TestRangeQuery, QueryResultKeyTest, TestMaxScoreQueryParser, SolrCoreCheckLockOnStartupTest, TermsComponentTest, WrapperMergePolicyFactoryTest, TestSSLRandomization, AliasIntegrationTest, TestApiFramework, TestTolerantUpdateProcessorRandomCloud, BasicZkTest, SpatialRPTFieldTypeTest, SolrCloudReportersTest, BitVectorTest, OpenExchangeRatesOrgProviderTest, CdcrRequestHandlerTest, HttpPartitionTest, TestUseDocValuesAsStored, TestClusterProperties, TestDynamicFieldResource, TestConfigOverlay, ZkSolrClientTest, TestCollationFieldDocValues, TestCloudManagedSchema, BaseCdcrDistributedZkTest, DistributedQueryElevationComponentTest, LukeRequestHandlerTest, TestRuleBasedAuthorizationPlugin, TestRandomFlRTGCloud, TestCloudRecovery, HdfsTlogReplayBufferedWhileIndexingTest, TestDocTermOrds, ZkStateWriterTest, TestMacroExpander, SolrMetricsIntegrationTest, AutoCommitTest, TestPerFieldSimilarityWithDefaultOverride, SimpleCollectionCreateDeleteTest, SolrCloudExampleTest, TestQueryUtils, TestSimpleQParserPlugin, InfixSuggestersTest, DistributedFacetPivotSmallTest, TestRawTransformer, TestLuceneMatchVersion, HLLUtilTest, DistributedIntervalFacetingTest, TestSQLHandler, TestComponentsName, TestPerFieldSimilarity, DistribDocExpirationUpdateProcessorTest, TestSolrConfigHandler, TestAuthorizationFramework, TestJsonRequest, TestPullReplica, DeleteStatusTest, TestStressLucene, TestSystemCollAutoCreate, TestLMJelinekMercerSimilarityFactory, TestFastOutputStream, CoreAdminRequestStatusTest, TestRequestForwarding, ReplicationFactorTest, EnumFieldTest, TestDistributedMissingSort, SolrGraphiteReporterTest, LeaderFailoverAfterPartitionTest, TestSystemIdResolver, TestSolrConfigHandlerCloud, TestManagedSynonymFilterFactory, TestMultiWordSynonyms, TestIntervalFaceting, TestSolr4Spatial, RuleEngineTest, SearchHandlerTest, RemoteQueryErrorTest, HdfsDirectoryTest, CdcrReplicationDistributedZkTest, TestComplexPhraseQParserPlugin, TestPushWriter, SubstringBytesRefFilterTest, IndexSchemaRuntimeFieldTest, SynonymTokenizerTest, MigrateRouteKeyTest, SolrMetricManagerTest, ConjunctionSolrSpellCheckerTest, DistributedFacetPivotLongTailTest, DocumentBuilderTest, TestTrackingShardHandlerFactory, AtomicUpdatesTest, TestFieldResource, DistributedFacetExistsSmallTest, TestEmbeddedSolrServerAdminHandler, TestNumericTerms32, SpellPossibilityIteratorTest, TestExceedMaxTermLength, TestCorePropertiesReload, TestRTGBase, TestCollectionsAPIViaSolrCloudCluster, SolrIndexMetricsTest, TestMinMaxOnMultiValuedField, HdfsChaosMonkeySafeLeaderTest, TestOverriddenPrefixQueryForCustomFieldType, TestSolrCloudWithDelegationTokens, SchemaVersionSpecificBehaviorTest, CursorMarkTest, TestCrossCoreJoin, TestPolicyCloud, ClassificationUpdateProcessorIntegrationTest, TestSchemaResource, TestFieldCacheReopen, HdfsWriteToMultipleCollectionsTest, TestFastLRUCache, ExitableDirectoryReaderTest, HdfsRecoveryZkTest, TestDelegationWithHadoopAuth, NodeMutatorTest, MergeStrategyTest, TestExclusionRuleCollectionAccess, SystemInfoHandlerTest, PreAnalyzedFieldTest, PropertiesRequestHandlerTest, DateRangeFieldTest, PingRequestHandlerTest, TestSolrCloudWithSecureImpersonation, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, OpenCloseCoreStressTest, LeaderElectionIntegrationTest, FullSolrCloudDistribCmdsTest, ZkCLITest, PeerSyncTest, ShowFileRequestHandlerTest, TestCoreDiscovery, AnalysisAfterCoreReloadTest, SignatureUpdateProcessorFactoryTest, TestExtendedDismaxParser, SpellCheckCollatorTest, SuggesterFSTTest, CoreAdminHandlerTest, SolrRequestParserTest, TestFoldingMultitermQuery, DocValuesTest, TestTrie, TestCSVLoader, TestPseudoReturnFields, TestUpdate, FieldMutatingUpdateProcessorTest, TestWordDelimiterFilterFactory, QueryEqualityTest, DirectUpdateHandlerOptimizeTest, StatelessScriptUpdateProcessorFactoryTest, SortByFunctionTest, DistanceFunctionTest, TestSolrDeletionPolicy1, SolrInfoBeanTest, XsltUpdateRequestHandlerTest, DebugComponentTest, CacheHeaderTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, TestReversedWildcardFilterFactory, DocumentAnalysisRequestHandlerTest, TestQueryTypes, TestOmitPositions, FileBasedSpellCheckerTest, TestValueSourceCache, TermVectorComponentTest, MoreLikeThisHandlerTest, RequiredFieldsTest, FieldAnalysisRequestHandlerTest, TestSolrQueryParser, FastVectorHighlighterTest, TestJmxIntegration, TestCollationField, ReturnFieldsTest, UpdateRequestProcessorFactoryTest, TestCSVResponseWriter, MBeansHandlerTest, TestMergePolicyConfig, MultiTermTest, SampleTest, TestDocSet, TestBinaryField, TestSearchPerf, MinimalSchemaTest, TestElisionMultitermQuery, OutputWriterTest, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, TestSolrCoreProperties, TestPostingsSolrHighlighter, NotRequiredUniqueKeyTest, TestCharFilters, TestXIncludeConfig, EchoParamsTest, TestSweetSpotSimilarityFactory, TestIBSimilarityFactory, ScriptEngineTest, DateMathParserTest, DateFieldTest, TestSolrJ, TestLRUCache, TestDocumentBuilder, ZkNodePropsTest, SliceStateTest, DistributedMLTComponentTest, CursorPagingTest, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, TestHighlightDedupGrouping, TestTolerantSearch, TestEmbeddedSolrServerSchemaAPI, ConnectionReuseTest, AssignBackwardCompatibilityTest, CdcrBootstrapTest, CdcrReplicationHandlerTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, CollectionStateFormat2Test, CollectionsAPIAsyncDistributedZkTest, DeleteLastCustomShardedReplicaTest, DeleteReplicaTest, MoveReplicaHDFSTest, MoveReplicaTest, MultiThreadedOCPTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerStatusTest, PeerSyncReplicationTest, SaslZkACLProviderTest, TestAuthenticationFramework, TestCloudJSONFacetJoinDomain, TestCloudPivotFacet, TestCloudPseudoReturnFields, TestCryptoKeys, TestDistribDocBasedVersion, TestHdfsCloudBackupRestore, TestLeaderElectionWithEmptyReplica, TestLeaderElectionZkExpiry, TestLeaderInitiatedRecoveryThread, TestLocalFSCloudBackupRestore, TestOnReconnectListenerSupport, TestPrepRecovery, TestReplicaProperties, TestShortCircuitedRequests, TestSizeLimitedDistributedMap, TestStressLiveNodes, TestTlogReplica, AutoScalingHandlerTest, HdfsBasicDistributedZkTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsCollectionsAPIDistributedZkTest, HdfsNNFailoverTest, HdfsRestartWhileUpdatingTest, HdfsSyncSliceTest, HdfsThreadLeakTest, HdfsUnloadDistributedZkTest, StressHdfsTest, BlobRepositoryCloudTest, CachingDirectoryFactoryTest, TestConfigSetProperties, TestDynamicLoading, TestSolrCoreSnapshots, TestHdfsBackupRestoreCore, TestSQLHandlerNonCloud, TestSolrConfigHandlerConcurrent, DistributedFacetPivotLargeTest, DistributedQueryComponentOptimizationTest, FacetPivotSmallTest, TestDistributedStatsComponentCardinality, SolrSlf4jReporterTest]
   [junit4] Completed [624/731 (3!)] on J1 in 6.29s, 1 test, 1 failure <<< FAILURES!

[...truncated 43764 lines...]