You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2019/04/08 03:16:01 UTC

[JENKINS] Lucene-Solr-Tests-8.x - Build # 108 - Unstable

Build: https://builds.apache.org/job/Lucene-Solr-Tests-8.x/108/

1 tests failed.
FAILED:  org.apache.solr.security.AuditLoggerIntegrationTest.testAsyncQueueDrain

Error Message:
Failed waiting for 3 callbacks after 5 seconds

Stack Trace:
java.lang.AssertionError: Failed waiting for 3 callbacks after 5 seconds
	at __randomizedtesting.SeedInfo.seed([D206A2E769293F9E:63195DC3D9EAAC41]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.apache.solr.security.AuditLoggerIntegrationTest.waitForAuditEventCallbacks(AuditLoggerIntegrationTest.java:229)
	at org.apache.solr.security.AuditLoggerIntegrationTest.waitForAuditEventCallbacks(AuditLoggerIntegrationTest.java:221)
	at org.apache.solr.security.AuditLoggerIntegrationTest.assertThreeAdminEvents(AuditLoggerIntegrationTest.java:252)
	at org.apache.solr.security.AuditLoggerIntegrationTest.testAsyncQueueDrain(AuditLoggerIntegrationTest.java:132)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13590 lines...]
   [junit4] Suite: org.apache.solr.security.AuditLoggerIntegrationTest
   [junit4]   2> 867894 INFO  (SUITE-AuditLoggerIntegrationTest-seed#[D206A2E769293F9E]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J2/temp/solr.security.AuditLoggerIntegrationTest_D206A2E769293F9E-001/init-core-data-001
   [junit4]   2> 867895 INFO  (SUITE-AuditLoggerIntegrationTest-seed#[D206A2E769293F9E]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 867896 INFO  (SUITE-AuditLoggerIntegrationTest-seed#[D206A2E769293F9E]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 867899 INFO  (TEST-AuditLoggerIntegrationTest.testAsyncQueueDrain-seed#[D206A2E769293F9E]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testAsyncQueueDrain
   [junit4]   2> 867902 INFO  (TEST-AuditLoggerIntegrationTest.testAsyncQueueDrain-seed#[D206A2E769293F9E]) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J2/temp/solr.security.AuditLoggerIntegrationTest_D206A2E769293F9E-001/tempDir-001
   [junit4]   2> 867902 INFO  (TEST-AuditLoggerIntegrationTest.testAsyncQueueDrain-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 867902 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 867902 INFO  (auditTestCallback-5713-thread-1) [    ] o.a.s.s.AuditLoggerIntegrationTest Listening for audit callbacks on on port 46601
   [junit4]   2> 867902 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 868002 INFO  (TEST-AuditLoggerIntegrationTest.testAsyncQueueDrain-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer start zk server on port:44237
   [junit4]   2> 868003 INFO  (TEST-AuditLoggerIntegrationTest.testAsyncQueueDrain-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:44237
   [junit4]   2> 868003 INFO  (TEST-AuditLoggerIntegrationTest.testAsyncQueueDrain-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 44237
   [junit4]   2> 868012 INFO  (zkConnectionManagerCallback-4217-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 868015 INFO  (zkConnectionManagerCallback-4219-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 868016 INFO  (zkConnectionManagerCallback-4221-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 868019 WARN  (jetty-launcher-4222-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 868019 INFO  (jetty-launcher-4222-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 868019 INFO  (jetty-launcher-4222-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 868020 INFO  (jetty-launcher-4222-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 868021 INFO  (jetty-launcher-4222-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 868021 INFO  (jetty-launcher-4222-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 868021 INFO  (jetty-launcher-4222-thread-1) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 868021 INFO  (jetty-launcher-4222-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@385625aa{/solr,null,AVAILABLE}
   [junit4]   2> 868023 INFO  (jetty-launcher-4222-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5d48f5b8{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:34634}
   [junit4]   2> 868023 INFO  (jetty-launcher-4222-thread-1) [    ] o.e.j.s.Server Started @868095ms
   [junit4]   2> 868023 INFO  (jetty-launcher-4222-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34634}
   [junit4]   2> 868024 ERROR (jetty-launcher-4222-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 868024 INFO  (jetty-launcher-4222-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 868024 INFO  (jetty-launcher-4222-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.1.0
   [junit4]   2> 868024 INFO  (jetty-launcher-4222-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 868024 INFO  (jetty-launcher-4222-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 868024 INFO  (jetty-launcher-4222-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-04-08T02:23:34.248Z
   [junit4]   2> 868025 INFO  (zkConnectionManagerCallback-4224-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 868026 INFO  (jetty-launcher-4222-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 868123 INFO  (jetty-launcher-4222-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 868124 WARN  (jetty-launcher-4222-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@13f61945[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 868128 WARN  (jetty-launcher-4222-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@126313c3[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 868129 INFO  (jetty-launcher-4222-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44237/solr
   [junit4]   2> 868131 INFO  (zkConnectionManagerCallback-4231-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 868134 INFO  (zkConnectionManagerCallback-4233-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 868216 INFO  (jetty-launcher-4222-thread-1) [n:127.0.0.1:34634_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:34634_solr
   [junit4]   2> 868217 INFO  (jetty-launcher-4222-thread-1) [n:127.0.0.1:34634_solr    ] o.a.s.c.Overseer Overseer (id=73094699109842949-127.0.0.1:34634_solr-n_0000000000) starting
   [junit4]   2> 868223 INFO  (zkConnectionManagerCallback-4240-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 868226 INFO  (jetty-launcher-4222-thread-1) [n:127.0.0.1:34634_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44237/solr ready
   [junit4]   2> 868227 INFO  (OverseerStateUpdate-73094699109842949-127.0.0.1:34634_solr-n_0000000000) [n:127.0.0.1:34634_solr    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:34634_solr
   [junit4]   2> 868228 INFO  (jetty-launcher-4222-thread-1) [n:127.0.0.1:34634_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34634_solr
   [junit4]   2> 868230 INFO  (zkCallback-4239-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 868230 INFO  (zkCallback-4232-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 868231 INFO  (jetty-launcher-4222-thread-1) [n:127.0.0.1:34634_solr    ] o.a.s.c.CoreContainer Initializing auditlogger plugin: solr.CallbackAuditLoggerPlugin
   [junit4]   2> 868354 INFO  (jetty-launcher-4222-thread-1) [n:127.0.0.1:34634_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 868417 INFO  (jetty-launcher-4222-thread-1) [n:127.0.0.1:34634_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34634.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@333291e3
   [junit4]   2> 868430 INFO  (jetty-launcher-4222-thread-1) [n:127.0.0.1:34634_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34634.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@333291e3
   [junit4]   2> 868431 INFO  (jetty-launcher-4222-thread-1) [n:127.0.0.1:34634_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34634.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@333291e3
   [junit4]   2> 868432 INFO  (jetty-launcher-4222-thread-1) [n:127.0.0.1:34634_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J2/temp/solr.security.AuditLoggerIntegrationTest_D206A2E769293F9E-001/tempDir-001/node1/.
   [junit4]   2> 868467 INFO  (zkConnectionManagerCallback-4247-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 868468 INFO  (TEST-AuditLoggerIntegrationTest.testAsyncQueueDrain-seed#[D206A2E769293F9E]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 868470 INFO  (TEST-AuditLoggerIntegrationTest.testAsyncQueueDrain-seed#[D206A2E769293F9E]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44237/solr ready
   [junit4]   2> 868475 INFO  (qtp509239235-13235) [n:127.0.0.1:34634_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 868475 INFO  (qtp509239235-13235) [n:127.0.0.1:34634_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 868496 INFO  (audit-4241-thread-1-processing-n:127.0.0.1:34634_solr) [n:127.0.0.1:34634_solr    ] o.a.s.s.CallbackAuditLoggerPlugin Sleeping for 100ms before sending callback
   [junit4]   2> 868498 INFO  (qtp509239235-13236) [n:127.0.0.1:34634_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterstatus with params action=CLUSTERSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 868504 INFO  (qtp509239235-13236) [n:127.0.0.1:34634_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=5
   [junit4]   2> 868505 INFO  (qtp509239235-13237) [n:127.0.0.1:34634_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :overseerstatus with params action=OVERSEERSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 868530 INFO  (qtp509239235-13237) [n:127.0.0.1:34634_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=OVERSEERSTATUS&wt=javabin&version=2} status=0 QTime=25
   [junit4]   2> 868535 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x103af3dd7680007, likely client has closed socket
   [junit4]   2> 868549 INFO  (jetty-closer-4248-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=539619292
   [junit4]   2> 868549 INFO  (jetty-closer-4248-thread-1) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:34634_solr
   [junit4]   2> 868550 INFO  (jetty-closer-4248-thread-1) [    ] o.a.s.c.CoreContainer Publish this node as DOWN...
   [junit4]   2> 868550 INFO  (jetty-closer-4248-thread-1) [    ] o.a.s.c.ZkController Publish node=127.0.0.1:34634_solr as DOWN
   [junit4]   2> 868551 INFO  (jetty-closer-4248-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 868557 INFO  (jetty-closer-4248-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 868557 INFO  (jetty-closer-4248-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4fda1c90: rootName = solr_34634, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@39d13480
   [junit4]   2> 868563 INFO  (jetty-closer-4248-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 868563 INFO  (jetty-closer-4248-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1986dcdc: rootName = solr_34634, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@6c44542
   [junit4]   2> 868566 INFO  (jetty-closer-4248-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 868566 INFO  (jetty-closer-4248-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@47de021e: rootName = solr_34634, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@491bab1c
   [junit4]   2> 868566 INFO  (jetty-closer-4248-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 868568 INFO  (closeThreadPool-4251-thread-2) [    ] o.a.s.c.Overseer Overseer (id=73094699109842949-127.0.0.1:34634_solr-n_0000000000) closing
   [junit4]   2> 868570 INFO  (OverseerAutoScalingTriggerThread-73094699109842949-127.0.0.1:34634_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 868580 INFO  (OverseerStateUpdate-73094699109842949-127.0.0.1:34634_solr-n_0000000000) [n:127.0.0.1:34634_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:34634_solr
   [junit4]   2> 868581 INFO  (closeThreadPool-4251-thread-1) [    ] o.a.s.c.Overseer Overseer (id=73094699109842949-127.0.0.1:34634_solr-n_0000000000) closing
   [junit4]   2> 868586 INFO  (jetty-closer-4248-thread-1) [    ] o.a.s.c.Overseer Overseer (id=73094699109842949-127.0.0.1:34634_solr-n_0000000000) closing
   [junit4]   2> 868600 INFO  (jetty-closer-4248-thread-1) [    ] o.a.s.s.AuditLoggerPlugin Async auditlogger queue still has 2 elements, sleeping to let it drain...
   [junit4]   2> 869406 INFO  (audit-4241-thread-1-processing-n:127.0.0.1:34634_solr) [n:127.0.0.1:34634_solr    ] o.a.s.s.CallbackAuditLoggerPlugin Sent audit callback {"requestUrl":"http://127.0.0.1:34634/solr/admin/collections","nodeName":"n:127.0.0.1:34634_solr","message":"Completed","level":"INFO","date":1554690214699,"clientIp":"127.0.0.1","collections":[],"headers":{"User-Agent":"Solr[org.apache.solr.client.solrj.impl.HttpSolrClient] 1.0","Connection":"keep-alive","Host":"127.0.0.1:34634"},"solrParams":{"action":["LIST"],"wt":["javabin"],"version":["2"]},"solrHost":"127.0.0.1","solrPort":34634,"solrIp":"127.0.0.1","resource":"/admin/collections","httpMethod":"GET","httpQueryString":"action=LIST&wt=javabin&version=2","eventType":"COMPLETED","requestType":"ADMIN","status":200,"qtime":0.0} to localhost:46601
   [junit4]   2> 869406 INFO  (audit-4241-thread-1-processing-n:127.0.0.1:34634_solr) [n:127.0.0.1:34634_solr    ] o.a.s.s.CallbackAuditLoggerPlugin Sleeping for 100ms before sending callback
   [junit4]   2> 869523 INFO  (audit-4241-thread-1-processing-n:127.0.0.1:34634_solr) [n:127.0.0.1:34634_solr    ] o.a.s.s.CallbackAuditLoggerPlugin Sent audit callback {"requestUrl":"http://127.0.0.1:34634/solr/admin/collections","nodeName":"n:127.0.0.1:34634_solr","message":"Completed","level":"INFO","date":1554690214728,"clientIp":"127.0.0.1","collections":[],"headers":{"User-Agent":"Solr[org.apache.solr.client.solrj.impl.HttpSolrClient] 1.0","Connection":"keep-alive","Host":"127.0.0.1:34634"},"solrParams":{"action":["CLUSTERSTATUS"],"wt":["javabin"],"version":["2"]},"solrHost":"127.0.0.1","solrPort":34634,"solrIp":"127.0.0.1","resource":"/admin/collections","httpMethod":"GET","httpQueryString":"action=CLUSTERSTATUS&wt=javabin&version=2","eventType":"COMPLETED","requestType":"ADMIN","status":200,"qtime":5.0} to localhost:46601
   [junit4]   2> 869523 INFO  (audit-4241-thread-1-processing-n:127.0.0.1:34634_solr) [n:127.0.0.1:34634_solr    ] o.a.s.s.CallbackAuditLoggerPlugin Sleeping for 100ms before sending callback
   [junit4]   2> 869570 INFO  (auditTestCallback-5713-thread-1) [    ] o.a.s.s.AuditLoggerIntegrationTest Received audit event for path /admin/collections
   [junit4]   2> 869579 INFO  (auditTestCallback-5713-thread-1) [    ] o.a.s.s.AuditLoggerIntegrationTest Received audit event for path /admin/collections
   [junit4]   2> 869600 INFO  (jetty-closer-4248-thread-1) [    ] o.a.s.s.AuditLoggerPlugin Shutting down async Auditlogger background thread(s)
   [junit4]   2> 869603 INFO  (jetty-closer-4248-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@5d48f5b8{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:0}
   [junit4]   2> 869604 INFO  (jetty-closer-4248-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@385625aa{/solr,null,UNAVAILABLE}
   [junit4]   2> 869604 INFO  (jetty-closer-4248-thread-1) [    ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 869608 INFO  (TEST-AuditLoggerIntegrationTest.testAsyncQueueDrain-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
   [junit4]   2> 869610 WARN  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	3	/solr/collections
   [junit4]   2> 
   [junit4]   2> 869618 INFO  (TEST-AuditLoggerIntegrationTest.testAsyncQueueDrain-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:44237
   [junit4]   2> 869618 INFO  (TEST-AuditLoggerIntegrationTest.testAsyncQueueDrain-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 44237
   [junit4]   2> 869629 INFO  (audit-4241-thread-1-processing-n:127.0.0.1:34634_solr) [n:127.0.0.1:34634_solr    ] o.a.s.s.CallbackAuditLoggerPlugin Sent audit callback {"requestUrl":"http://127.0.0.1:34634/solr/admin/collections","nodeName":"n:127.0.0.1:34634_solr","message":"Completed","level":"INFO","date":1554690214754,"clientIp":"127.0.0.1","collections":[],"headers":{"User-Agent":"Solr[org.apache.solr.client.solrj.impl.HttpSolrClient] 1.0","Connection":"keep-alive","Host":"127.0.0.1:34634"},"solrParams":{"action":["OVERSEERSTATUS"],"wt":["javabin"],"version":["2"]},"solrHost":"127.0.0.1","solrPort":34634,"solrIp":"127.0.0.1","resource":"/admin/collections","httpMethod":"GET","httpQueryString":"action=OVERSEERSTATUS&wt=javabin&version=2","eventType":"COMPLETED","requestType":"ADMIN","status":200,"qtime":25.0} to localhost:46601
   [junit4]   2> 874631 INFO  (TEST-AuditLoggerIntegrationTest.testAsyncQueueDrain-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
   [junit4]   2> 874631 INFO  (TEST-AuditLoggerIntegrationTest.testAsyncQueueDrain-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:44237
   [junit4]   2> 874631 INFO  (TEST-AuditLoggerIntegrationTest.testAsyncQueueDrain-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 44237
   [junit4]   2> 874632 INFO  (TEST-AuditLoggerIntegrationTest.testAsyncQueueDrain-seed#[D206A2E769293F9E]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testAsyncQueueDrain
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=AuditLoggerIntegrationTest -Dtests.method=testAsyncQueueDrain -Dtests.seed=D206A2E769293F9E -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es-US -Dtests.timezone=Atlantic/South_Georgia -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 6.74s J2 | AuditLoggerIntegrationTest.testAsyncQueueDrain <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Failed waiting for 3 callbacks after 5 seconds
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D206A2E769293F9E:63195DC3D9EAAC41]:0)
   [junit4]    > 	at org.apache.solr.security.AuditLoggerIntegrationTest.waitForAuditEventCallbacks(AuditLoggerIntegrationTest.java:229)
   [junit4]    > 	at org.apache.solr.security.AuditLoggerIntegrationTest.waitForAuditEventCallbacks(AuditLoggerIntegrationTest.java:221)
   [junit4]    > 	at org.apache.solr.security.AuditLoggerIntegrationTest.assertThreeAdminEvents(AuditLoggerIntegrationTest.java:252)
   [junit4]    > 	at org.apache.solr.security.AuditLoggerIntegrationTest.testAsyncQueueDrain(AuditLoggerIntegrationTest.java:132)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 874639 INFO  (TEST-AuditLoggerIntegrationTest.testAsync-seed#[D206A2E769293F9E]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testAsync
   [junit4]   2> 874639 INFO  (auditTestCallback-5728-thread-1) [    ] o.a.s.s.AuditLoggerIntegrationTest Listening for audit callbacks on on port 32801
   [junit4]   2> 874640 INFO  (TEST-AuditLoggerIntegrationTest.testAsync-seed#[D206A2E769293F9E]) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J2/temp/solr.security.AuditLoggerIntegrationTest_D206A2E769293F9E-001/tempDir-002
   [junit4]   2> 874640 INFO  (TEST-AuditLoggerIntegrationTest.testAsync-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 874641 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 874641 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 874741 INFO  (TEST-AuditLoggerIntegrationTest.testAsync-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer start zk server on port:34680
   [junit4]   2> 874741 INFO  (TEST-AuditLoggerIntegrationTest.testAsync-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:34680
   [junit4]   2> 874741 INFO  (TEST-AuditLoggerIntegrationTest.testAsync-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 34680
   [junit4]   2> 874746 INFO  (zkConnectionManagerCallback-4254-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 874749 INFO  (zkConnectionManagerCallback-4256-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 874751 INFO  (zkConnectionManagerCallback-4258-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 874759 WARN  (jetty-launcher-4259-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 874760 INFO  (jetty-launcher-4259-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 874760 INFO  (jetty-launcher-4259-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 874760 INFO  (jetty-launcher-4259-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 874761 INFO  (jetty-launcher-4259-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 874761 INFO  (jetty-launcher-4259-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 874761 INFO  (jetty-launcher-4259-thread-1) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 874761 INFO  (jetty-launcher-4259-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7d1906fd{/solr,null,AVAILABLE}
   [junit4]   2> 874762 INFO  (jetty-launcher-4259-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@683cb64a{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:32862}
   [junit4]   2> 874762 INFO  (jetty-launcher-4259-thread-1) [    ] o.e.j.s.Server Started @874834ms
   [junit4]   2> 874762 INFO  (jetty-launcher-4259-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=32862}
   [junit4]   2> 874762 ERROR (jetty-launcher-4259-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 874762 INFO  (jetty-launcher-4259-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 874762 INFO  (jetty-launcher-4259-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.1.0
   [junit4]   2> 874762 INFO  (jetty-launcher-4259-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 874762 INFO  (jetty-launcher-4259-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 874762 INFO  (jetty-launcher-4259-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-04-08T02:23:40.986Z
   [junit4]   2> 874765 INFO  (zkConnectionManagerCallback-4261-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 874765 INFO  (jetty-launcher-4259-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 874877 INFO  (jetty-launcher-4259-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 874878 WARN  (jetty-launcher-4259-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@cefee2a[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 874882 WARN  (jetty-launcher-4259-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@153533bb[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 874883 INFO  (jetty-launcher-4259-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34680/solr
   [junit4]   2> 874885 INFO  (zkConnectionManagerCallback-4268-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 874888 INFO  (zkConnectionManagerCallback-4270-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 874971 INFO  (jetty-launcher-4259-thread-1) [n:127.0.0.1:32862_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:32862_solr
   [junit4]   2> 874972 INFO  (jetty-launcher-4259-thread-1) [n:127.0.0.1:32862_solr    ] o.a.s.c.Overseer Overseer (id=73094699551424517-127.0.0.1:32862_solr-n_0000000000) starting
   [junit4]   2> 874978 INFO  (zkConnectionManagerCallback-4277-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 874981 INFO  (jetty-launcher-4259-thread-1) [n:127.0.0.1:32862_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34680/solr ready
   [junit4]   2> 874981 INFO  (OverseerStateUpdate-73094699551424517-127.0.0.1:32862_solr-n_0000000000) [n:127.0.0.1:32862_solr    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:32862_solr
   [junit4]   2> 874982 INFO  (jetty-launcher-4259-thread-1) [n:127.0.0.1:32862_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32862_solr
   [junit4]   2> 874983 INFO  (OverseerStateUpdate-73094699551424517-127.0.0.1:32862_solr-n_0000000000) [n:127.0.0.1:32862_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 874983 INFO  (zkCallback-4276-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 874985 INFO  (jetty-launcher-4259-thread-1) [n:127.0.0.1:32862_solr    ] o.a.s.c.CoreContainer Initializing auditlogger plugin: solr.CallbackAuditLoggerPlugin
   [junit4]   2> 875009 INFO  (jetty-launcher-4259-thread-1) [n:127.0.0.1:32862_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 875050 INFO  (jetty-launcher-4259-thread-1) [n:127.0.0.1:32862_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_32862.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@333291e3
   [junit4]   2> 875064 INFO  (jetty-launcher-4259-thread-1) [n:127.0.0.1:32862_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_32862.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@333291e3
   [junit4]   2> 875064 INFO  (jetty-launcher-4259-thread-1) [n:127.0.0.1:32862_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_32862.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@333291e3
   [junit4]   2> 875065 INFO  (jetty-launcher-4259-thread-1) [n:127.0.0.1:32862_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J2/temp/solr.security.AuditLoggerIntegrationTest_D206A2E769293F9E-001/tempDir-002/node1/.
   [junit4]   2> 875092 INFO  (zkConnectionManagerCallback-4284-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 875094 INFO  (TEST-AuditLoggerIntegrationTest.testAsync-seed#[D206A2E769293F9E]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 875095 INFO  (TEST-AuditLoggerIntegrationTest.testAsync-seed#[D206A2E769293F9E]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34680/solr ready
   [junit4]   2> 875099 INFO  (qtp180625928-13308) [n:127.0.0.1:32862_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 875099 INFO  (qtp180625928-13308) [n:127.0.0.1:32862_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 875099 INFO  (qtp180625928-13309) [n:127.0.0.1:32862_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterstatus with params action=CLUSTERSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 875100 INFO  (qtp180625928-13309) [n:127.0.0.1:32862_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 875101 INFO  (qtp180625928-13310) [n:127.0.0.1:32862_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :overseerstatus with params action=OVERSEERSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 875126 INFO  (qtp180625928-13310) [n:127.0.0.1:32862_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=OVERSEERSTATUS&wt=javabin&version=2} status=0 QTime=25
   [junit4]   2> 875132 INFO  (audit-4278-thread-1-processing-n:127.0.0.1:32862_solr) [n:127.0.0.1:32862_solr    ] o.a.s.s.CallbackAuditLoggerPlugin Sent audit callback {"requestUrl":"http://127.0.0.1:32862/solr/admin/collections","nodeName":"n:127.0.0.1:32862_solr","message":"Completed","level":"INFO","date":1554690221323,"clientIp":"127.0.0.1","collections":[],"headers":{"User-Agent":"Solr[org.apache.solr.client.solrj.impl.HttpSolrClient] 1.0","Connection":"keep-alive","Host":"127.0.0.1:32862"},"solrParams":{"action":["LIST"],"wt":["javabin"],"version":["2"]},"solrHost":"127.0.0.1","solrPort":32862,"solrIp":"127.0.0.1","resource":"/admin/collections","httpMethod":"GET","httpQueryString":"action=LIST&wt=javabin&version=2","eventType":"COMPLETED","requestType":"ADMIN","status":200,"qtime":0.0} to localhost:32801
   [junit4]   2> 875142 INFO  (auditTestCallback-5728-thread-1) [    ] o.a.s.s.AuditLoggerIntegrationTest Received audit event for path /admin/collections
   [junit4]   2> 875153 INFO  (audit-4278-thread-1-processing-n:127.0.0.1:32862_solr) [n:127.0.0.1:32862_solr    ] o.a.s.s.CallbackAuditLoggerPlugin Sent audit callback {"requestUrl":"http://127.0.0.1:32862/solr/admin/collections","nodeName":"n:127.0.0.1:32862_solr","message":"Completed","level":"INFO","date":1554690221324,"clientIp":"127.0.0.1","collections":[],"headers":{"User-Agent":"Solr[org.apache.solr.client.solrj.impl.HttpSolrClient] 1.0","Connection":"keep-alive","Host":"127.0.0.1:32862"},"solrParams":{"action":["CLUSTERSTATUS"],"wt":["javabin"],"version":["2"]},"solrHost":"127.0.0.1","solrPort":32862,"solrIp":"127.0.0.1","resource":"/admin/collections","httpMethod":"GET","httpQueryString":"action=CLUSTERSTATUS&wt=javabin&version=2","eventType":"COMPLETED","requestType":"ADMIN","status":200,"qtime":0.0} to localhost:32801
   [junit4]   2> 875158 INFO  (auditTestCallback-5728-thread-1) [    ] o.a.s.s.AuditLoggerIntegrationTest Received audit event for path /admin/collections
   [junit4]   2> 875175 INFO  (auditTestCallback-5728-thread-1) [    ] o.a.s.s.AuditLoggerIntegrationTest Received audit event for path /admin/collections
   [junit4]   2> 875177 INFO  (audit-4278-thread-1-processing-n:127.0.0.1:32862_solr) [n:127.0.0.1:32862_solr    ] o.a.s.s.CallbackAuditLoggerPlugin Sent audit callback {"requestUrl":"http://127.0.0.1:32862/solr/admin/collections","nodeName":"n:127.0.0.1:32862_solr","message":"Completed","level":"INFO","date":1554690221351,"clientIp":"127.0.0.1","collections":[],"headers":{"User-Agent":"Solr[org.apache.solr.client.solrj.impl.HttpSolrClient] 1.0","Connection":"keep-alive","Host":"127.0.0.1:32862"},"solrParams":{"action":["OVERSEERSTATUS"],"wt":["javabin"],"version":["2"]},"solrHost":"127.0.0.1","solrPort":32862,"solrIp":"127.0.0.1","resource":"/admin/collections","httpMethod":"GET","httpQueryString":"action=OVERSEERSTATUS&wt=javabin&version=2","eventType":"COMPLETED","requestType":"ADMIN","status":200,"qtime":25.0} to localhost:32801
   [junit4]   2> 875233 INFO  (jetty-closer-4285-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1470949247
   [junit4]   2> 875233 INFO  (jetty-closer-4285-thread-1) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:32862_solr
   [junit4]   2> 875234 INFO  (jetty-closer-4285-thread-1) [    ] o.a.s.c.CoreContainer Publish this node as DOWN...
   [junit4]   2> 875234 INFO  (jetty-closer-4285-thread-1) [    ] o.a.s.c.ZkController Publish node=127.0.0.1:32862_solr as DOWN
   [junit4]   2> 875236 INFO  (jetty-closer-4285-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 875236 INFO  (jetty-closer-4285-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 875236 INFO  (jetty-closer-4285-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1f49cbce: rootName = solr_32862, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@7efd1f80
   [junit4]   2> 875242 INFO  (jetty-closer-4285-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 875242 INFO  (jetty-closer-4285-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@556694e9: rootName = solr_32862, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@6c44542
   [junit4]   2> 875245 INFO  (jetty-closer-4285-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 875245 INFO  (jetty-closer-4285-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4420401b: rootName = solr_32862, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@491bab1c
   [junit4]   2> 875245 INFO  (jetty-closer-4285-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 875246 INFO  (closeThreadPool-4288-thread-1) [    ] o.a.s.c.Overseer Overseer (id=73094699551424517-127.0.0.1:32862_solr-n_0000000000) closing
   [junit4]   2> 875247 INFO  (OverseerStateUpdate-73094699551424517-127.0.0.1:32862_solr-n_0000000000) [n:127.0.0.1:32862_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:32862_solr
   [junit4]   2> 875248 INFO  (OverseerAutoScalingTriggerThread-73094699551424517-127.0.0.1:32862_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 875254 INFO  (jetty-closer-4285-thread-1) [    ] o.a.s.c.Overseer Overseer (id=73094699551424517-127.0.0.1:32862_solr-n_0000000000) closing
   [junit4]   2> 875254 INFO  (closeThreadPool-4288-thread-2) [    ] o.a.s.c.Overseer Overseer (id=73094699551424517-127.0.0.1:32862_solr-n_0000000000) closing
   [junit4]   2> 875256 INFO  (jetty-closer-4285-thread-1) [    ] o.a.s.s.AuditLoggerPlugin Shutting down async Auditlogger background thread(s)
   [junit4]   2> 875256 WARN  (audit-4278-thread-1-processing-n:127.0.0.1:32862_solr) [n:127.0.0.1:32862_solr    ] o.a.s.s.AuditLoggerPlugin Interrupted while waiting for next audit log event
   [junit4]   2> 875256 INFO  (jetty-closer-4285-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@683cb64a{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:0}
   [junit4]   2> 875257 INFO  (jetty-closer-4285-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7d1906fd{/solr,null,UNAVAILABLE}
   [junit4]   2> 875257 INFO  (jetty-closer-4285-thread-1) [    ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 875257 INFO  (TEST-AuditLoggerIntegrationTest.testAsync-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
   [junit4]   2> 875259 WARN  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	3	/solr/collections
   [junit4]   2> 
   [junit4]   2> 875262 INFO  (TEST-AuditLoggerIntegrationTest.testAsync-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:34680
   [junit4]   2> 875262 INFO  (TEST-AuditLoggerIntegrationTest.testAsync-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 34680
   [junit4]   2> 875262 INFO  (TEST-AuditLoggerIntegrationTest.testAsync-seed#[D206A2E769293F9E]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testAsync
   [junit4]   2> 875267 INFO  (TEST-AuditLoggerIntegrationTest.testMuteAdminListCollections-seed#[D206A2E769293F9E]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testMuteAdminListCollections
   [junit4]   2> 875267 INFO  (auditTestCallback-5743-thread-1) [    ] o.a.s.s.AuditLoggerIntegrationTest Listening for audit callbacks on on port 36542
   [junit4]   2> 875268 INFO  (TEST-AuditLoggerIntegrationTest.testMuteAdminListCollections-seed#[D206A2E769293F9E]) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J2/temp/solr.security.AuditLoggerIntegrationTest_D206A2E769293F9E-001/tempDir-003
   [junit4]   2> 875268 INFO  (TEST-AuditLoggerIntegrationTest.testMuteAdminListCollections-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 875269 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 875269 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 875369 INFO  (TEST-AuditLoggerIntegrationTest.testMuteAdminListCollections-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer start zk server on port:44489
   [junit4]   2> 875369 INFO  (TEST-AuditLoggerIntegrationTest.testMuteAdminListCollections-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:44489
   [junit4]   2> 875369 INFO  (TEST-AuditLoggerIntegrationTest.testMuteAdminListCollections-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 44489
   [junit4]   2> 875373 INFO  (zkConnectionManagerCallback-4290-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 875376 INFO  (zkConnectionManagerCallback-4292-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 875377 INFO  (zkConnectionManagerCallback-4294-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 875380 WARN  (jetty-launcher-4295-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 875380 INFO  (jetty-launcher-4295-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 875380 INFO  (jetty-launcher-4295-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 875380 INFO  (jetty-launcher-4295-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 875381 INFO  (jetty-launcher-4295-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 875381 INFO  (jetty-launcher-4295-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 875381 INFO  (jetty-launcher-4295-thread-1) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 875382 INFO  (jetty-launcher-4295-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6abcd448{/solr,null,AVAILABLE}
   [junit4]   2> 875382 INFO  (jetty-launcher-4295-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@9caa1c1{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:44094}
   [junit4]   2> 875382 INFO  (jetty-launcher-4295-thread-1) [    ] o.e.j.s.Server Started @875454ms
   [junit4]   2> 875382 INFO  (jetty-launcher-4295-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=44094}
   [junit4]   2> 875383 ERROR (jetty-launcher-4295-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 875383 INFO  (jetty-launcher-4295-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 875383 INFO  (jetty-launcher-4295-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.1.0
   [junit4]   2> 875383 INFO  (jetty-launcher-4295-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 875383 INFO  (jetty-launcher-4295-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 875383 INFO  (jetty-launcher-4295-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-04-08T02:23:41.607Z
   [junit4]   2> 875384 INFO  (zkConnectionManagerCallback-4297-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 875385 INFO  (jetty-launcher-4295-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 875531 INFO  (jetty-launcher-4295-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 875532 WARN  (jetty-launcher-4295-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@4e08dafc[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 875536 WARN  (jetty-launcher-4295-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@712a4fd0[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 875536 INFO  (jetty-launcher-4295-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44489/solr
   [junit4]   2> 875538 INFO  (zkConnectionManagerCallback-4304-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 875541 INFO  (zkConnectionManagerCallback-4306-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 875619 INFO  (jetty-launcher-4295-thread-1) [n:127.0.0.1:44094_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:44094_solr
   [junit4]   2> 875619 INFO  (jetty-launcher-4295-thread-1) [n:127.0.0.1:44094_solr    ] o.a.s.c.Overseer Overseer (id=73094699592581125-127.0.0.1:44094_solr-n_0000000000) starting
   [junit4]   2> 875626 INFO  (zkConnectionManagerCallback-4313-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 875628 INFO  (jetty-launcher-4295-thread-1) [n:127.0.0.1:44094_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44489/solr ready
   [junit4]   2> 875629 INFO  (OverseerStateUpdate-73094699592581125-127.0.0.1:44094_solr-n_0000000000) [n:127.0.0.1:44094_solr    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:44094_solr
   [junit4]   2> 875629 INFO  (jetty-launcher-4295-thread-1) [n:127.0.0.1:44094_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44094_solr
   [junit4]   2> 875631 INFO  (OverseerStateUpdate-73094699592581125-127.0.0.1:44094_solr-n_0000000000) [n:127.0.0.1:44094_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 875631 INFO  (zkCallback-4312-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 875632 INFO  (jetty-launcher-4295-thread-1) [n:127.0.0.1:44094_solr    ] o.a.s.c.CoreContainer Initializing auditlogger plugin: solr.CallbackAuditLoggerPlugin
   [junit4]   2> 875658 INFO  (jetty-launcher-4295-thread-1) [n:127.0.0.1:44094_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 875682 INFO  (jetty-launcher-4295-thread-1) [n:127.0.0.1:44094_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44094.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@333291e3
   [junit4]   2> 875695 INFO  (jetty-launcher-4295-thread-1) [n:127.0.0.1:44094_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44094.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@333291e3
   [junit4]   2> 875695 INFO  (jetty-launcher-4295-thread-1) [n:127.0.0.1:44094_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44094.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@333291e3
   [junit4]   2> 875696 INFO  (jetty-launcher-4295-thread-1) [n:127.0.0.1:44094_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J2/temp/solr.security.AuditLoggerIntegrationTest_D206A2E769293F9E-001/tempDir-003/node1/.
   [junit4]   2> 875722 INFO  (zkConnectionManagerCallback-4319-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 875726 INFO  (TEST-AuditLoggerIntegrationTest.testMuteAdminListCollections-seed#[D206A2E769293F9E]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 875727 INFO  (TEST-AuditLoggerIntegrationTest.testMuteAdminListCollections-seed#[D206A2E769293F9E]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44489/solr ready
   [junit4]   2> 875732 INFO  (qtp1140548439-13380) [n:127.0.0.1:44094_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 875732 INFO  (qtp1140548439-13380) [n:127.0.0.1:44094_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 875734 INFO  (qtp1140548439-13381) [n:127.0.0.1:44094_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterstatus with params action=CLUSTERSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 875734 INFO  (qtp1140548439-13381) [n:127.0.0.1:44094_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CLUSTERSTATUS&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 875769 INFO  (auditTestCallback-5743-thread-1) [    ] o.a.s.s.AuditLoggerIntegrationTest Received audit event for path /admin/collections
   [junit4]   2> 875770 INFO  (qtp1140548439-13381) [n:127.0.0.1:44094_solr    ] o.a.s.s.CallbackAuditLoggerPlugin Sent audit callback {"requestUrl":"http://127.0.0.1:44094/solr/admin/collections","nodeName":"n:127.0.0.1:44094_solr","message":"Completed","level":"INFO","date":1554690221959,"clientIp":"127.0.0.1","collections":[],"headers":{"User-Agent":"Solr[org.apache.solr.client.solrj.impl.HttpSolrClient] 1.0","Connection":"keep-alive","Host":"127.0.0.1:44094"},"solrParams":{"action":["CLUSTERSTATUS"],"wt":["javabin"],"version":["2"]},"solrHost":"127.0.0.1","solrPort":44094,"solrIp":"127.0.0.1","resource":"/admin/collections","httpMethod":"GET","httpQueryString":"action=CLUSTERSTATUS&wt=javabin&version=2","eventType":"COMPLETED","requestType":"ADMIN","status":200,"qtime":1.0} to localhost:36542
   [junit4]   2> 875771 INFO  (qtp1140548439-13382) [n:127.0.0.1:44094_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :overseerstatus with params action=OVERSEERSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 875790 INFO  (qtp1140548439-13382) [n:127.0.0.1:44094_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=OVERSEERSTATUS&wt=javabin&version=2} status=0 QTime=19
   [junit4]   2> 875805 INFO  (auditTestCallback-5743-thread-1) [    ] o.a.s.s.AuditLoggerIntegrationTest Received audit event for path /admin/collections
   [junit4]   2> 875846 INFO  (qtp1140548439-13382) [n:127.0.0.1:44094_solr    ] o.a.s.s.CallbackAuditLoggerPlugin Sent audit callback {"requestUrl":"http://127.0.0.1:44094/solr/admin/collections","nodeName":"n:127.0.0.1:44094_solr","message":"Completed","level":"INFO","date":1554690222014,"clientIp":"127.0.0.1","collections":[],"headers":{"User-Agent":"Solr[org.apache.solr.client.solrj.impl.HttpSolrClient] 1.0","Connection":"keep-alive","Host":"127.0.0.1:44094"},"solrParams":{"action":["OVERSEERSTATUS"],"wt":["javabin"],"version":["2"]},"solrHost":"127.0.0.1","solrPort":44094,"solrIp":"127.0.0.1","resource":"/admin/collections","httpMethod":"GET","httpQueryString":"action=OVERSEERSTATUS&wt=javabin&version=2","eventType":"COMPLETED","requestType":"ADMIN","status":200,"qtime":19.0} to localhost:36542
   [junit4]   2> 875857 INFO  (jetty-closer-4320-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1697053361
   [junit4]   2> 875857 INFO  (jetty-closer-4320-thread-1) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:44094_solr
   [junit4]   2> 875857 INFO  (jetty-closer-4320-thread-1) [    ] o.a.s.c.CoreContainer Publish this node as DOWN...
   [junit4]   2> 875857 INFO  (jetty-closer-4320-thread-1) [    ] o.a.s.c.ZkController Publish node=127.0.0.1:44094_solr as DOWN
   [junit4]   2> 875859 INFO  (jetty-closer-4320-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 875860 INFO  (jetty-closer-4320-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 875860 INFO  (jetty-closer-4320-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@41c467b9: rootName = solr_44094, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@723d664f
   [junit4]   2> 875866 INFO  (jetty-closer-4320-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 875866 INFO  (jetty-closer-4320-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@77670325: rootName = solr_44094, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@6c44542
   [junit4]   2> 875869 INFO  (jetty-closer-4320-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 875869 INFO  (jetty-closer-4320-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1ab495b2: rootName = solr_44094, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@491bab1c
   [junit4]   2> 875869 INFO  (jetty-closer-4320-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 875872 INFO  (closeThreadPool-4323-thread-1) [    ] o.a.s.c.Overseer Overseer (id=73094699592581125-127.0.0.1:44094_solr-n_0000000000) closing
   [junit4]   2> 875872 INFO  (OverseerStateUpdate-73094699592581125-127.0.0.1:44094_solr-n_0000000000) [n:127.0.0.1:44094_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:44094_solr
   [junit4]   2> 875885 INFO  (OverseerAutoScalingTriggerThread-73094699592581125-127.0.0.1:44094_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 875892 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x103af3df42e0005, likely client has closed socket
   [junit4]   2> 875893 INFO  (jetty-closer-4320-thread-1) [    ] o.a.s.c.Overseer Overseer (id=73094699592581125-127.0.0.1:44094_solr-n_0000000000) closing
   [junit4]   2> 875893 INFO  (closeThreadPool-4323-thread-2) [    ] o.a.s.c.Overseer Overseer (id=73094699592581125-127.0.0.1:44094_solr-n_0000000000) closing
   [junit4]   2> 875900 INFO  (jetty-closer-4320-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@9caa1c1{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:0}
   [junit4]   2> 875900 INFO  (jetty-closer-4320-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6abcd448{/solr,null,UNAVAILABLE}
   [junit4]   2> 875900 INFO  (jetty-closer-4320-thread-1) [    ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 875904 INFO  (TEST-AuditLoggerIntegrationTest.testMuteAdminListCollections-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
   [junit4]   2> 875906 WARN  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	3	/solr/collections
   [junit4]   2> 
   [junit4]   2> 875908 INFO  (TEST-AuditLoggerIntegrationTest.testMuteAdminListCollections-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:44489
   [junit4]   2> 875908 INFO  (TEST-AuditLoggerIntegrationTest.testMuteAdminListCollections-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 44489
   [junit4]   2> 875909 INFO  (TEST-AuditLoggerIntegrationTest.testMuteAdminListCollections-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
   [junit4]   2> 875909 INFO  (TEST-AuditLoggerIntegrationTest.testMuteAdminListCollections-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:44489
   [junit4]   2> 875909 INFO  (TEST-AuditLoggerIntegrationTest.testMuteAdminListCollections-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 44489
   [junit4]   2> 875909 INFO  (TEST-AuditLoggerIntegrationTest.testMuteAdminListCollections-seed#[D206A2E769293F9E]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testMuteAdminListCollections
   [junit4]   2> 875913 INFO  (TEST-AuditLoggerIntegrationTest.auth-seed#[D206A2E769293F9E]) [    ] o.a.s.SolrTestCaseJ4 ###Starting auth
   [junit4]   2> 875914 INFO  (auditTestCallback-5758-thread-1) [    ] o.a.s.s.AuditLoggerIntegrationTest Listening for audit callbacks on on port 43875
   [junit4]   2> 875915 INFO  (TEST-AuditLoggerIntegrationTest.auth-seed#[D206A2E769293F9E]) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J2/temp/solr.security.AuditLoggerIntegrationTest_D206A2E769293F9E-001/tempDir-004
   [junit4]   2> 875915 INFO  (TEST-AuditLoggerIntegrationTest.auth-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 875919 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 875919 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 876019 INFO  (TEST-AuditLoggerIntegrationTest.auth-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer start zk server on port:33038
   [junit4]   2> 876019 INFO  (TEST-AuditLoggerIntegrationTest.auth-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:33038
   [junit4]   2> 876019 INFO  (TEST-AuditLoggerIntegrationTest.auth-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 33038
   [junit4]   2> 876030 INFO  (zkConnectionManagerCallback-4326-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 876034 INFO  (zkConnectionManagerCallback-4328-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 876035 INFO  (zkConnectionManagerCallback-4330-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 876038 WARN  (jetty-launcher-4331-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 876039 INFO  (jetty-launcher-4331-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 876039 INFO  (jetty-launcher-4331-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 876039 INFO  (jetty-launcher-4331-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 876040 INFO  (jetty-launcher-4331-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 876040 INFO  (jetty-launcher-4331-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 876040 INFO  (jetty-launcher-4331-thread-1) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 876040 INFO  (jetty-launcher-4331-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@12152f6f{/solr,null,AVAILABLE}
   [junit4]   2> 876041 INFO  (jetty-launcher-4331-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1db5a2a{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:43155}
   [junit4]   2> 876041 INFO  (jetty-launcher-4331-thread-1) [    ] o.e.j.s.Server Started @876113ms
   [junit4]   2> 876041 INFO  (jetty-launcher-4331-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43155}
   [junit4]   2> 876041 ERROR (jetty-launcher-4331-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 876041 INFO  (jetty-launcher-4331-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 876041 INFO  (jetty-launcher-4331-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.1.0
   [junit4]   2> 876041 INFO  (jetty-launcher-4331-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 876041 INFO  (jetty-launcher-4331-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 876041 INFO  (jetty-launcher-4331-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-04-08T02:23:42.265Z
   [junit4]   2> 876044 INFO  (zkConnectionManagerCallback-4333-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 876045 INFO  (jetty-launcher-4331-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 876302 INFO  (jetty-launcher-4331-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 876321 WARN  (jetty-launcher-4331-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@7da4d243[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 876338 WARN  (jetty-launcher-4331-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@2f843185[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 876338 INFO  (jetty-launcher-4331-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33038/solr
   [junit4]   2> 876347 INFO  (zkConnectionManagerCallback-4340-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 876355 INFO  (zkConnectionManagerCallback-4342-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 876438 INFO  (jetty-launcher-4331-thread-1) [n:127.0.0.1:43155_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43155_solr
   [junit4]   2> 876438 INFO  (jetty-launcher-4331-thread-1) [n:127.0.0.1:43155_solr    ] o.a.s.c.Overseer Overseer (id=73094699635179525-127.0.0.1:43155_solr-n_0000000000) starting
   [junit4]   2> 876449 INFO  (zkConnectionManagerCallback-4349-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 876452 INFO  (jetty-launcher-4331-thread-1) [n:127.0.0.1:43155_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33038/solr ready
   [junit4]   2> 876453 INFO  (OverseerStateUpdate-73094699635179525-127.0.0.1:43155_solr-n_0000000000) [n:127.0.0.1:43155_solr    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:43155_solr
   [junit4]   2> 876453 INFO  (jetty-launcher-4331-thread-1) [n:127.0.0.1:43155_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43155_solr
   [junit4]   2> 876456 INFO  (jetty-launcher-4331-thread-1) [n:127.0.0.1:43155_solr    ] o.a.s.c.CoreContainer Initializing authorization plugin: solr.RuleBasedAuthorizationPlugin
   [junit4]   2> 876468 INFO  (zkCallback-4341-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 876482 INFO  (zkCallback-4348-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 876500 INFO  (jetty-launcher-4331-thread-1) [n:127.0.0.1:43155_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: solr.BasicAuthPlugin
   [junit4]   2> 876540 INFO  (jetty-launcher-4331-thread-1) [n:127.0.0.1:43155_solr    ] o.a.s.c.CoreContainer Initializing auditlogger plugin: solr.CallbackAuditLoggerPlugin
   [junit4]   2> 876565 INFO  (jetty-launcher-4331-thread-1) [n:127.0.0.1:43155_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 876591 INFO  (jetty-launcher-4331-thread-1) [n:127.0.0.1:43155_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43155.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@333291e3
   [junit4]   2> 876604 INFO  (jetty-launcher-4331-thread-1) [n:127.0.0.1:43155_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43155.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@333291e3
   [junit4]   2> 876604 INFO  (jetty-launcher-4331-thread-1) [n:127.0.0.1:43155_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43155.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@333291e3
   [junit4]   2> 876605 INFO  (jetty-launcher-4331-thread-1) [n:127.0.0.1:43155_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J2/temp/solr.security.AuditLoggerIntegrationTest_D206A2E769293F9E-001/tempDir-004/node1/.
   [junit4]   2> 876734 INFO  (zkConnectionManagerCallback-4355-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 876735 INFO  (TEST-AuditLoggerIntegrationTest.auth-seed#[D206A2E769293F9E]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 876736 INFO  (TEST-AuditLoggerIntegrationTest.auth-seed#[D206A2E769293F9E]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33038/solr ready
   [junit4]   2> 876756 INFO  (qtp1833492446-13449) [n:127.0.0.1:43155_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 876756 INFO  (qtp1833492446-13449) [n:127.0.0.1:43155_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 876770 INFO  (qtp1833492446-13449) [n:127.0.0.1:43155_solr    ] o.a.s.s.CallbackAuditLoggerPlugin Sent audit callback {"requestUrl":"http://127.0.0.1:43155/solr/admin/collections","nodeName":"n:127.0.0.1:43155_solr","message":"Completed","level":"INFO","date":1554690222980,"clientIp":"127.0.0.1","collections":[],"headers":{"User-Agent":"Solr[org.apache.solr.client.solrj.impl.HttpSolrClient] 1.0","Connection":"keep-alive","Host":"127.0.0.1:43155"},"solrParams":{"action":["LIST"],"wt":["javabin"],"version":["2"]},"solrHost":"127.0.0.1","solrPort":43155,"solrIp":"127.0.0.1","resource":"/admin/collections","httpMethod":"GET","httpQueryString":"action=LIST&wt=javabin&version=2","eventType":"COMPLETED","requestType":"ADMIN","status":200,"qtime":1.0} to localhost:43875
   [junit4]   2> 876774 INFO  (auditTestCallback-5758-thread-1) [    ] o.a.s.s.AuditLoggerIntegrationTest Received audit event for path /admin/collections
   [junit4]   2> 876813 INFO  (qtp1833492446-13449) [n:127.0.0.1:43155_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 876813 INFO  (qtp1833492446-13449) [n:127.0.0.1:43155_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 876857 INFO  (auditTestCallback-5758-thread-1) [    ] o.a.s.s.AuditLoggerIntegrationTest Received audit event for path /admin/collections
   [junit4]   2> 876868 INFO  (qtp1833492446-13449) [n:127.0.0.1:43155_solr    ] o.a.s.s.CallbackAuditLoggerPlugin Sent audit callback {"requestUrl":"http://127.0.0.1:43155/solr/admin/collections","nodeName":"n:127.0.0.1:43155_solr","message":"Completed","level":"INFO","date":1554690223037,"username":"solr","clientIp":"127.0.0.1","collections":[],"headers":{"Authorization":"Basic c29scjpTb2xyUm9ja3M=","User-Agent":"Solr[org.apache.solr.client.solrj.impl.HttpSolrClient] 1.0","Connection":"keep-alive","Host":"127.0.0.1:43155"},"solrParams":{"action":["LIST"],"wt":["javabin"],"version":["2"]},"solrHost":"127.0.0.1","solrPort":43155,"solrIp":"127.0.0.1","resource":"/admin/collections","httpMethod":"GET","httpQueryString":"action=LIST&wt=javabin&version=2","eventType":"COMPLETED","requestType":"ADMIN","status":200,"qtime":0.0} to localhost:43875
   [junit4]   2> 876869 INFO  (qtp1833492446-13449) [n:127.0.0.1:43155_solr    ] o.a.s.s.RuleBasedAuthorizationPlugin request has come without principal. failed permission {
   [junit4]   2>   "name":"collection-admin-edit",
   [junit4]   2>   "role":"admin"} 
   [junit4]   2> 876883 INFO  (qtp1833492446-13449) [n:127.0.0.1:43155_solr    ] o.a.s.s.CallbackAuditLoggerPlugin Sent audit callback {"requestUrl":"http://127.0.0.1:43155/solr/admin/collections","nodeName":"n:127.0.0.1:43155_solr","message":"Rejected","level":"WARN","date":1554690223093,"clientIp":"127.0.0.1","collections":["test"],"headers":{"User-Agent":"Solr[org.apache.solr.client.solrj.impl.HttpSolrClient] 1.0","Connection":"keep-alive","Host":"127.0.0.1:43155"},"solrParams":{"pullReplicas":["0"],"name":["test"],"nrtReplicas":["1"],"action":["CREATE"],"numShards":["1"],"tlogReplicas":["0"],"wt":["javabin"],"version":["2"]},"solrHost":"127.0.0.1","solrPort":43155,"solrIp":"127.0.0.1","resource":"/admin/collections","httpMethod":"GET","httpQueryString":"action=CREATE&name=test&numShards=1&nrtReplicas=1&pullReplicas=0&tlogReplicas=0&wt=javabin&version=2","eventType":"REJECTED","requestType":"ADMIN","status":401,"qtime":-1.0} to localhost:43875
   [junit4]   2> 876890 INFO  (auditTestCallback-5758-thread-1) [    ] o.a.s.s.AuditLoggerIntegrationTest Received audit event for path /admin/collections
   [junit4]   2> 876883 INFO  (qtp1833492446-13449) [n:127.0.0.1:43155_solr    ] o.a.s.s.HttpSolrCall USER_REQUIRED auth header null context : userPrincipal: [null] type: [ADMIN], collections: [test,], Path: [/admin/collections] path : /admin/collections params :pullReplicas=0&name=test&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 
   [junit4]   1> {"requestUrl":"http://127.0.0.1:43155/solr/admin/collections","nodeName":"n:127.0.0.1:43155_solr","message":"Completed","level":"INFO","date":1554690223037,"username":"solr","clientIp":"127.0.0.1","collections":[],"headers":{"Authorization":"Basic c29scjpTb2xyUm9ja3M=","User-Agent":"Solr[org.apache.solr.client.solrj.impl.HttpSolrClient] 1.0","Connection":"keep-alive","Host":"127.0.0.1:43155"},"solrParams":{"action":["LIST"],"wt":["javabin"],"version":["2"]},"solrHost":"127.0.0.1","solrPort":43155,"solrIp":"127.0.0.1","resource":"/admin/collections","httpMethod":"GET","httpQueryString":"action=LIST&wt=javabin&version=2","eventType":"COMPLETED","requestType":"ADMIN","status":200,"qtime":0.0}
   [junit4]   2> 876960 INFO  (qtp1833492446-13449) [n:127.0.0.1:43155_solr    ] o.a.s.s.CallbackAuditLoggerPlugin Sent audit callback {"requestUrl":"http://127.0.0.1:43155/solr/admin/collections","nodeName":"n:127.0.0.1:43155_solr","message":"Unauthorized","level":"WARN","date":1554690223151,"clientIp":"127.0.0.1","collections":["test"],"headers":{"User-Agent":"Solr[org.apache.solr.client.solrj.impl.HttpSolrClient] 1.0","Connection":"keep-alive","Host":"127.0.0.1:43155"},"solrParams":{"pullReplicas":["0"],"name":["test"],"nrtReplicas":["1"],"action":["CREATE"],"numShards":["1"],"tlogReplicas":["0"],"wt":["javabin"],"version":["2"]},"solrHost":"127.0.0.1","solrPort":43155,"solrIp":"127.0.0.1","resource":"/admin/collections","httpMethod":"GET","httpQueryString":"action=CREATE&name=test&numShards=1&nrtReplicas=1&pullReplicas=0&tlogReplicas=0&wt=javabin&version=2","eventType":"UNAUTHORIZED","requestType":"ADMIN","status":403,"qtime":-1.0} to localhost:43875
   [junit4]   2> 876995 INFO  (qtp1833492446-13449) [n:127.0.0.1:43155_solr    ] o.a.s.s.CallbackAuditLoggerPlugin Sent audit callback {"requestUrl":"http://127.0.0.1:43155/solr/admin/collections","nodeName":"n:127.0.0.1:43155_solr","message":"Rejected","level":"WARN","date":1554690223195,"clientIp":"127.0.0.1","headers":{"Authorization":"Basic c29scjp3cm9uZ1BX","User-Agent":"Solr[org.apache.solr.client.solrj.impl.HttpSolrClient] 1.0","Connection":"keep-alive","Host":"127.0.0.1:43155"},"solrParams":{},"solrHost":"127.0.0.1","solrPort":43155,"solrIp":"127.0.0.1","resource":"/solr","httpMethod":"GET","httpQueryString":"action=CREATE&name=test&numShards=1&nrtReplicas=1&pullReplicas=0&tlogReplicas=0&wt=javabin&version=2","eventType":"REJECTED","requestType":"UNKNOWN","status":401,"qtime":-1.0} to localhost:43875
   [junit4]   2> 877001 INFO  (auditTestCallback-5758-thread-1) [    ] o.a.s.s.AuditLoggerIntegrationTest Received audit event for path /admin/collections
   [junit4]   2> 877023 INFO  (auditTestCallback-5758-thread-1) [    ] o.a.s.s.AuditLoggerIntegrationTest Received audit event for path /solr
   [junit4]   2> 877090 INFO  (jetty-closer-4356-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1927507791
   [junit4]   2> 877090 INFO  (jetty-closer-4356-thread-1) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:43155_solr
   [junit4]   2> 877092 INFO  (jetty-closer-4356-thread-1) [    ] o.a.s.c.CoreContainer Publish this node as DOWN...
   [junit4]   2> 877092 INFO  (jetty-closer-4356-thread-1) [    ] o.a.s.c.ZkController Publish node=127.0.0.1:43155_solr as DOWN
   [junit4]   2> 877094 INFO  (jetty-closer-4356-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 877096 INFO  (jetty-closer-4356-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 877096 INFO  (jetty-closer-4356-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2efb2523: rootName = solr_43155, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@6284563b
   [junit4]   2> 877102 INFO  (jetty-closer-4356-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 877102 INFO  (jetty-closer-4356-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1f3fbb57: rootName = solr_43155, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@6c44542
   [junit4]   2> 877113 INFO  (jetty-closer-4356-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 877113 INFO  (jetty-closer-4356-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1d0eeab5: rootName = solr_43155, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@491bab1c
   [junit4]   2> 877114 INFO  (jetty-closer-4356-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 877115 INFO  (closeThreadPool-4359-thread-2) [    ] o.a.s.c.Overseer Overseer (id=73094699635179525-127.0.0.1:43155_solr-n_0000000000) closing
   [junit4]   2> 877116 INFO  (OverseerAutoScalingTriggerThread-73094699635179525-127.0.0.1:43155_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 877116 INFO  (OverseerStateUpdate-73094699635179525-127.0.0.1:43155_solr-n_0000000000) [n:127.0.0.1:43155_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:43155_solr
   [junit4]   2> 877118 INFO  (zkCallback-4348-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 877120 INFO  (jetty-closer-4356-thread-1) [    ] o.a.s.c.Overseer Overseer (id=73094699635179525-127.0.0.1:43155_solr-n_0000000000) closing
   [junit4]   2> 877120 INFO  (closeThreadPool-4359-thread-1) [    ] o.a.s.c.Overseer Overseer (id=73094699635179525-127.0.0.1:43155_solr-n_0000000000) closing
   [junit4]   2> 877121 INFO  (jetty-closer-4356-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@1db5a2a{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:0}
   [junit4]   2> 877121 INFO  (jetty-closer-4356-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@12152f6f{/solr,null,UNAVAILABLE}
   [junit4]   2> 877121 INFO  (jetty-closer-4356-thread-1) [    ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 877122 INFO  (TEST-AuditLoggerIntegrationTest.auth-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
   [junit4]   2> 877124 WARN  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	3	/solr/collections
   [junit4]   2> 
   [junit4]   2> 877128 INFO  (TEST-AuditLoggerIntegrationTest.auth-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:33038
   [junit4]   2> 877128 INFO  (TEST-AuditLoggerIntegrationTest.auth-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 33038
   [junit4]   2> 877128 INFO  (TEST-AuditLoggerIntegrationTest.auth-seed#[D206A2E769293F9E]) [    ] o.a.s.SolrTestCaseJ4 ###Ending auth
   [junit4]   2> 877133 INFO  (TEST-AuditLoggerIntegrationTest.testQueuedTimeMetric-seed#[D206A2E769293F9E]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testQueuedTimeMetric
   [junit4]   2> 877133 INFO  (auditTestCallback-5773-thread-1) [    ] o.a.s.s.AuditLoggerIntegrationTest Listening for audit callbacks on on port 41106
   [junit4]   2> 877134 INFO  (TEST-AuditLoggerIntegrationTest.testQueuedTimeMetric-seed#[D206A2E769293F9E]) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J2/temp/solr.security.AuditLoggerIntegrationTest_D206A2E769293F9E-001/tempDir-005
   [junit4]   2> 877135 INFO  (TEST-AuditLoggerIntegrationTest.testQueuedTimeMetric-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 877135 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 877135 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 877235 INFO  (TEST-AuditLoggerIntegrationTest.testQueuedTimeMetric-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer start zk server on port:36913
   [junit4]   2> 877235 INFO  (TEST-AuditLoggerIntegrationTest.testQueuedTimeMetric-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:36913
   [junit4]   2> 877235 INFO  (TEST-AuditLoggerIntegrationTest.testQueuedTimeMetric-seed#[D206A2E769293F9E]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 36913
   [junit4]   2> 877239 INFO  (zkConnectionManagerCallback-4361-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 877242 INFO  (zkConnectionManagerCallback-4363-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 877243 INFO  (zkConnectionManagerCallback-4365-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 877246 WARN  (jetty-launcher-4366-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 877246 INFO  (jetty-launcher-4366-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 877246 INFO  (jetty-launcher-4366-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 877246 INFO  (jetty-launcher-4366-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 877247 INFO  (jetty-launcher-4366-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 877247 INFO  (jetty-launcher-4366-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 877247 INFO  (jetty-launcher-4366-thread-1) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 877248 INFO  (jetty-launcher-4366-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@46eac18d{/solr,null,AVAILABLE}
   [junit4]   2> 877248 INFO  (jetty-launcher-4366-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@e03435d{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:46737}
   [junit4]   2> 877248 INFO  (jetty-launcher-4366-thread-1) [    ] o.e.j.s.Server Started @877320ms
   [junit4]   2> 877248 INFO  (jetty-launcher-4366-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46737}
   [junit4]   2> 877249 ERROR (jetty-launcher-4366-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 877249 INFO  (jetty-launcher-4366-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 877249 INFO  (jetty-launcher-4366-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.1.0
   [junit4]   2> 877249 INFO  (jetty-launcher-4366-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 877249 INFO  (jetty-launcher-4366-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 877249 INFO  (jetty-launcher-4366-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-04-08T02:23:43.473Z
   [junit4]   2> 877251 INFO  (zkConnectionManagerCallback-4368-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 877252 INFO  (jetty-launcher-4366-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 877714 INFO  (jetty-launcher-4366-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 877716 WARN  (jetty-launcher-4366-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@1564ffd2[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 877720 WARN  (jetty-launcher-4366-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@78511285[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 877720 INFO  (jetty-launcher-4366-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36913/solr
   [junit4]   2> 877723 INFO  (zkConnectionManagerCallback-4375-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 877726 INFO  (zkConnectionManagerCallback-4377-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   

[...truncated too long message...]

d.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/null315988145
     [copy] Copying 240 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/null315988145
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/null315988145

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] 	confs: [default]
[ivy:cachepath] 	found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] 	found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] 	found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] 	found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] 	found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] 	found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 27ms :: artifacts dl 2ms
	---------------------------------------------------------------------
	|                  |            modules            ||   artifacts   |
	|       conf       | number| search|dwnlded|evicted|| number|dwnlded|
	---------------------------------------------------------------------
	|      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
	---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 107 minutes 56 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2847)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2726)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2707)
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3086)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.FilePath.act(FilePath.java:1072)
		at hudson.FilePath.act(FilePath.java:1061)
		at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
		at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
		at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
		at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
		at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
		at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
		at hudson.model.Build$BuildExecution.post2(Build.java:186)
		at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
		at hudson.model.Run.execute(Run.java:1835)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3088)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath.act(FilePath.java:1074)
	at hudson.FilePath.act(FilePath.java:1061)
	at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
	at hudson.model.Build$BuildExecution.post2(Build.java:186)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
	at hudson.model.Run.execute(Run.java:1835)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)

[JENKINS] Lucene-Solr-Tests-8.x - Build # 110 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-8.x/110/

1 tests failed.
FAILED:  org.apache.solr.security.BasicAuthIntegrationTest.testBasicAuth

Error Message:
Expected metric minimums for prefix SECURITY./authentication.: {failMissingCredentials=2, authenticated=19, passThrough=9, failWrongCredentials=1, requests=31, errors=0}, but got: {failMissingCredentials=2, authenticated=17, passThrough=12, totalTime=41748139, failWrongCredentials=1, requestTimes=361, requests=32, errors=0}

Stack Trace:
java.lang.AssertionError: Expected metric minimums for prefix SECURITY./authentication.: {failMissingCredentials=2, authenticated=19, passThrough=9, failWrongCredentials=1, requests=31, errors=0}, but got: {failMissingCredentials=2, authenticated=17, passThrough=12, totalTime=41748139, failWrongCredentials=1, requestTimes=361, requests=32, errors=0}
	at __randomizedtesting.SeedInfo.seed([BE7C305C7E19584C:212464EDA4ADB36]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.assertTrue(Assert.java:41)
	at org.apache.solr.cloud.SolrCloudAuthTestCase.assertAuthMetricsMinimums(SolrCloudAuthTestCase.java:129)
	at org.apache.solr.cloud.SolrCloudAuthTestCase.assertAuthMetricsMinimums(SolrCloudAuthTestCase.java:83)
	at org.apache.solr.security.BasicAuthIntegrationTest.testBasicAuth(BasicAuthIntegrationTest.java:306)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12996 lines...]
   [junit4] Suite: org.apache.solr.security.BasicAuthIntegrationTest
   [junit4]   2> 709772 INFO  (SUITE-BasicAuthIntegrationTest-seed#[BE7C305C7E19584C]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_BE7C305C7E19584C-001/init-core-data-001
   [junit4]   2> 709772 WARN  (SUITE-BasicAuthIntegrationTest-seed#[BE7C305C7E19584C]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=24 numCloses=24
   [junit4]   2> 709772 INFO  (SUITE-BasicAuthIntegrationTest-seed#[BE7C305C7E19584C]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 709774 INFO  (SUITE-BasicAuthIntegrationTest-seed#[BE7C305C7E19584C]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 709886 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testBasicAuth
   [junit4]   2> 709886 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_BE7C305C7E19584C-001/tempDir-001
   [junit4]   2> 709887 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 709911 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 709911 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 710077 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.c.ZkTestServer start zk server on port:45637
   [junit4]   2> 710077 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:45637
   [junit4]   2> 710077 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 45637
   [junit4]   2> 710150 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0: ZooKeeperServer not running
   [junit4]   2> 712285 INFO  (zkConnectionManagerCallback-2046-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 712290 INFO  (zkConnectionManagerCallback-2048-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 712310 INFO  (zkConnectionManagerCallback-2050-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 712364 WARN  (jetty-launcher-2051-thread-3) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 712364 INFO  (jetty-launcher-2051-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 712364 INFO  (jetty-launcher-2051-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 712364 INFO  (jetty-launcher-2051-thread-3) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 712365 INFO  (jetty-launcher-2051-thread-3) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 712365 INFO  (jetty-launcher-2051-thread-3) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 712365 INFO  (jetty-launcher-2051-thread-3) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 712366 INFO  (jetty-launcher-2051-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1588b8f8{/solr,null,AVAILABLE}
   [junit4]   2> 712367 WARN  (jetty-launcher-2051-thread-3) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@2a3514fd[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 712367 INFO  (jetty-launcher-2051-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@39caf33d{SSL,[ssl, http/1.1]}{127.0.0.1:44906}
   [junit4]   2> 712367 INFO  (jetty-launcher-2051-thread-3) [    ] o.e.j.s.Server Started @712464ms
   [junit4]   2> 712367 INFO  (jetty-launcher-2051-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=44906}
   [junit4]   2> 712367 ERROR (jetty-launcher-2051-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 712367 INFO  (jetty-launcher-2051-thread-3) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 712367 INFO  (jetty-launcher-2051-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.1.0
   [junit4]   2> 712367 INFO  (jetty-launcher-2051-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 712367 INFO  (jetty-launcher-2051-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 712367 INFO  (jetty-launcher-2051-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-04-09T01:02:12.322Z
   [junit4]   2> 712370 INFO  (zkConnectionManagerCallback-2053-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 712384 WARN  (jetty-launcher-2051-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 712387 INFO  (jetty-launcher-2051-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 712397 WARN  (jetty-launcher-2051-thread-2) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 712419 INFO  (jetty-launcher-2051-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 712419 INFO  (jetty-launcher-2051-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 712419 INFO  (jetty-launcher-2051-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 712424 INFO  (jetty-launcher-2051-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 712424 INFO  (jetty-launcher-2051-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 712424 INFO  (jetty-launcher-2051-thread-2) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 712432 INFO  (jetty-launcher-2051-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 712432 INFO  (jetty-launcher-2051-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 712432 INFO  (jetty-launcher-2051-thread-2) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 712433 INFO  (jetty-launcher-2051-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4b51247b{/solr,null,AVAILABLE}
   [junit4]   2> 712433 WARN  (jetty-launcher-2051-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@4520a5fe[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 712434 INFO  (jetty-launcher-2051-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@33d54eb8{SSL,[ssl, http/1.1]}{127.0.0.1:42743}
   [junit4]   2> 712434 INFO  (jetty-launcher-2051-thread-2) [    ] o.e.j.s.Server Started @712531ms
   [junit4]   2> 712434 INFO  (jetty-launcher-2051-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=42743}
   [junit4]   2> 712434 ERROR (jetty-launcher-2051-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 712434 INFO  (jetty-launcher-2051-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 712434 INFO  (jetty-launcher-2051-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.1.0
   [junit4]   2> 712434 INFO  (jetty-launcher-2051-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 712434 INFO  (jetty-launcher-2051-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 712434 INFO  (jetty-launcher-2051-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-04-09T01:02:12.389Z
   [junit4]   2> 712461 INFO  (jetty-launcher-2051-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 712461 INFO  (jetty-launcher-2051-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 712461 INFO  (jetty-launcher-2051-thread-1) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 712476 INFO  (jetty-launcher-2051-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@61926aeb{/solr,null,AVAILABLE}
   [junit4]   2> 712476 WARN  (jetty-launcher-2051-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@64a064b8[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 712488 INFO  (jetty-launcher-2051-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@363c9538{SSL,[ssl, http/1.1]}{127.0.0.1:38331}
   [junit4]   2> 712488 INFO  (jetty-launcher-2051-thread-1) [    ] o.e.j.s.Server Started @712586ms
   [junit4]   2> 712488 INFO  (jetty-launcher-2051-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38331}
   [junit4]   2> 712489 ERROR (jetty-launcher-2051-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 712489 INFO  (jetty-launcher-2051-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 712489 INFO  (jetty-launcher-2051-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.1.0
   [junit4]   2> 712489 INFO  (jetty-launcher-2051-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 712489 INFO  (jetty-launcher-2051-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 712489 INFO  (jetty-launcher-2051-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-04-09T01:02:12.444Z
   [junit4]   2> 712489 INFO  (zkConnectionManagerCallback-2055-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 712490 INFO  (jetty-launcher-2051-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 712525 INFO  (zkConnectionManagerCallback-2057-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 712570 INFO  (jetty-launcher-2051-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 713573 INFO  (jetty-launcher-2051-thread-2) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 713573 WARN  (jetty-launcher-2051-thread-2) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 713586 WARN  (jetty-launcher-2051-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@3af6afa5[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 713604 WARN  (jetty-launcher-2051-thread-2) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 713605 WARN  (jetty-launcher-2051-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@64fe68ac[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 713606 INFO  (jetty-launcher-2051-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45637/solr
   [junit4]   2> 713619 INFO  (zkConnectionManagerCallback-2066-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 713772 INFO  (zkConnectionManagerCallback-2068-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 714162 INFO  (jetty-launcher-2051-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 714162 WARN  (jetty-launcher-2051-thread-1) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 714230 WARN  (jetty-launcher-2051-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@1f6d3f58[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 714232 WARN  (jetty-launcher-2051-thread-1) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 714252 WARN  (jetty-launcher-2051-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@17ffe8dd[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 714253 INFO  (jetty-launcher-2051-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45637/solr
   [junit4]   2> 714263 INFO  (zkConnectionManagerCallback-2076-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 714280 INFO  (zkConnectionManagerCallback-2078-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 714321 INFO  (jetty-launcher-2051-thread-1) [n:127.0.0.1:38331_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:38331_solr
   [junit4]   2> 714322 INFO  (jetty-launcher-2051-thread-1) [n:127.0.0.1:38331_solr    ] o.a.s.c.Overseer Overseer (id=74489578049110025-127.0.0.1:38331_solr-n_0000000000) starting
   [junit4]   2> 714518 INFO  (zkConnectionManagerCallback-2085-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 714603 INFO  (jetty-launcher-2051-thread-1) [n:127.0.0.1:38331_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45637/solr ready
   [junit4]   2> 714606 INFO  (jetty-launcher-2051-thread-2) [n:127.0.0.1:42743_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42743_solr
   [junit4]   2> 714621 INFO  (jetty-launcher-2051-thread-1) [n:127.0.0.1:38331_solr    ] o.a.s.c.ZkController Publish node=127.0.0.1:38331_solr as DOWN
   [junit4]   2> 714622 INFO  (jetty-launcher-2051-thread-1) [n:127.0.0.1:38331_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 714622 INFO  (jetty-launcher-2051-thread-1) [n:127.0.0.1:38331_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38331_solr
   [junit4]   2> 714631 INFO  (OverseerStateUpdate-74489578049110025-127.0.0.1:38331_solr-n_0000000000) [n:127.0.0.1:38331_solr    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:38331_solr
   [junit4]   2> 714653 INFO  (zkCallback-2077-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 714663 INFO  (zkCallback-2084-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 714663 INFO  (zkCallback-2067-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 714748 INFO  (jetty-launcher-2051-thread-3) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 714748 WARN  (jetty-launcher-2051-thread-3) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 714756 INFO  (zkConnectionManagerCallback-2090-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 714785 INFO  (jetty-launcher-2051-thread-1) [n:127.0.0.1:38331_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 714788 INFO  (jetty-launcher-2051-thread-2) [n:127.0.0.1:42743_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 714821 INFO  (jetty-launcher-2051-thread-2) [n:127.0.0.1:42743_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45637/solr ready
   [junit4]   2> 714846 WARN  (jetty-launcher-2051-thread-3) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@3779d87d[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 714854 INFO  (jetty-launcher-2051-thread-2) [n:127.0.0.1:42743_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 714882 WARN  (jetty-launcher-2051-thread-3) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 714903 WARN  (jetty-launcher-2051-thread-3) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@74cfebeb[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 714903 INFO  (jetty-launcher-2051-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45637/solr
   [junit4]   2> 715085 INFO  (zkConnectionManagerCallback-2098-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 715123 INFO  (jetty-launcher-2051-thread-2) [n:127.0.0.1:42743_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42743.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@216a5b5
   [junit4]   2> 715176 INFO  (jetty-launcher-2051-thread-2) [n:127.0.0.1:42743_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42743.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@216a5b5
   [junit4]   2> 715176 INFO  (jetty-launcher-2051-thread-2) [n:127.0.0.1:42743_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42743.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@216a5b5
   [junit4]   2> 715177 INFO  (jetty-launcher-2051-thread-2) [n:127.0.0.1:42743_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_BE7C305C7E19584C-001/tempDir-001/node1/.
   [junit4]   2> 715283 INFO  (zkConnectionManagerCallback-2100-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 715738 INFO  (jetty-launcher-2051-thread-1) [n:127.0.0.1:38331_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38331.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@216a5b5
   [junit4]   2> 715774 INFO  (jetty-launcher-2051-thread-1) [n:127.0.0.1:38331_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38331.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@216a5b5
   [junit4]   2> 715775 INFO  (jetty-launcher-2051-thread-1) [n:127.0.0.1:38331_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38331.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@216a5b5
   [junit4]   2> 715776 INFO  (jetty-launcher-2051-thread-1) [n:127.0.0.1:38331_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_BE7C305C7E19584C-001/tempDir-001/node3/.
   [junit4]   2> 715779 INFO  (jetty-launcher-2051-thread-3) [n:127.0.0.1:44906_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 715808 INFO  (jetty-launcher-2051-thread-3) [n:127.0.0.1:44906_solr    ] o.a.s.c.ZkController Publish node=127.0.0.1:44906_solr as DOWN
   [junit4]   2> 715809 INFO  (jetty-launcher-2051-thread-3) [n:127.0.0.1:44906_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 715809 INFO  (jetty-launcher-2051-thread-3) [n:127.0.0.1:44906_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44906_solr
   [junit4]   2> 715855 INFO  (zkCallback-2089-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 715903 INFO  (zkCallback-2067-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 715903 INFO  (zkCallback-2099-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 715903 INFO  (zkCallback-2077-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 715903 INFO  (zkCallback-2084-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 716256 INFO  (zkConnectionManagerCallback-2107-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 716257 INFO  (jetty-launcher-2051-thread-3) [n:127.0.0.1:44906_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 716258 INFO  (jetty-launcher-2051-thread-3) [n:127.0.0.1:44906_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45637/solr ready
   [junit4]   2> 716272 INFO  (jetty-launcher-2051-thread-3) [n:127.0.0.1:44906_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 716411 INFO  (jetty-launcher-2051-thread-3) [n:127.0.0.1:44906_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44906.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@216a5b5
   [junit4]   2> 716447 INFO  (jetty-launcher-2051-thread-3) [n:127.0.0.1:44906_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44906.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@216a5b5
   [junit4]   2> 716447 INFO  (jetty-launcher-2051-thread-3) [n:127.0.0.1:44906_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44906.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@216a5b5
   [junit4]   2> 716448 INFO  (jetty-launcher-2051-thread-3) [n:127.0.0.1:44906_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_BE7C305C7E19584C-001/tempDir-001/node2/.
   [junit4]   2> 716770 INFO  (zkConnectionManagerCallback-2113-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 716792 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 716793 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45637/solr ready
   [junit4]   2> 717015 INFO  (qtp128994539-5505) [n:127.0.0.1:42743_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf&name=authCollection&nrtReplicas=1&action=CREATE&numShards=3&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 717228 INFO  (OverseerThreadFactory-1434-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr    ] o.a.s.c.a.c.CreateCollectionCmd Create collection authCollection
   [junit4]   2> 717429 INFO  (OverseerStateUpdate-74489578049110025-127.0.0.1:38331_solr-n_0000000000) [n:127.0.0.1:38331_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"authCollection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"authCollection_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:42743/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 717449 INFO  (OverseerStateUpdate-74489578049110025-127.0.0.1:38331_solr-n_0000000000) [n:127.0.0.1:38331_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"authCollection",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"authCollection_shard2_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:44906/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 717536 INFO  (OverseerStateUpdate-74489578049110025-127.0.0.1:38331_solr-n_0000000000) [n:127.0.0.1:38331_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"authCollection",
   [junit4]   2>   "shard":"shard3",
   [junit4]   2>   "core":"authCollection_shard3_replica_n4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:38331/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 718014 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr    x:authCollection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=authCollection_shard1_replica_n1&action=CREATE&numShards=3&collection=authCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 718015 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr    x:authCollection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 718110 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr    x:authCollection_shard2_replica_n2] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=authCollection_shard2_replica_n2&action=CREATE&numShards=3&collection=authCollection&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 718236 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr    x:authCollection_shard3_replica_n4] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node6&collection.configName=conf&newCollection=true&name=authCollection_shard3_replica_n4&action=CREATE&numShards=3&collection=authCollection&shard=shard3&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 719147 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.1.0
   [junit4]   2> 719449 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.s.IndexSchema [authCollection_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 719450 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.1.0
   [junit4]   2> 719452 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 719452 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'authCollection_shard1_replica_n1' using configuration from collection authCollection, trusted=true
   [junit4]   2> 719523 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_42743.solr.core.authCollection.shard1.replica_n1' (registry 'solr.core.authCollection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@216a5b5
   [junit4]   2> 719525 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.SolrCore [[authCollection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_BE7C305C7E19584C-001/tempDir-001/node1/authCollection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_BE7C305C7E19584C-001/tempDir-001/node1/./authCollection_shard1_replica_n1/data/]
   [junit4]   2> 719539 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.s.IndexSchema [authCollection_shard2_replica_n2] Schema name=minimal
   [junit4]   2> 719540 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.1.0
   [junit4]   2> 719777 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 719777 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'authCollection_shard2_replica_n2' using configuration from collection authCollection, trusted=true
   [junit4]   2> 719778 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_44906.solr.core.authCollection.shard2.replica_n2' (registry 'solr.core.authCollection.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@216a5b5
   [junit4]   2> 719778 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.SolrCore [[authCollection_shard2_replica_n2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_BE7C305C7E19584C-001/tempDir-001/node2/authCollection_shard2_replica_n2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_BE7C305C7E19584C-001/tempDir-001/node2/./authCollection_shard2_replica_n2/data/]
   [junit4]   2> 719846 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.s.IndexSchema [authCollection_shard3_replica_n4] Schema name=minimal
   [junit4]   2> 719848 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 719848 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'authCollection_shard3_replica_n4' using configuration from collection authCollection, trusted=true
   [junit4]   2> 719848 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38331.solr.core.authCollection.shard3.replica_n4' (registry 'solr.core.authCollection.shard3.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@216a5b5
   [junit4]   2> 719849 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SolrCore [[authCollection_shard3_replica_n4] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_BE7C305C7E19584C-001/tempDir-001/node3/authCollection_shard3_replica_n4], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_BE7C305C7E19584C-001/tempDir-001/node3/./authCollection_shard3_replica_n4/data/]
   [junit4]   2> 720486 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 720486 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 720488 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 720488 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 720506 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 720506 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 720508 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 720508 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 720522 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@584587f7[authCollection_shard1_replica_n1] main]
   [junit4]   2> 720522 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@56b871e3[authCollection_shard2_replica_n2] main]
   [junit4]   2> 720523 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 720524 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 720524 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 720524 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1630296332544507904
   [junit4]   2> 720542 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 720543 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 720543 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 720543 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1630296332564430848
   [junit4]   2> 720644 INFO  (searcherExecutor-1451-thread-1-processing-n:127.0.0.1:44906_solr x:authCollection_shard2_replica_n2 c:authCollection s:shard2 r:core_node5) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.SolrCore [authCollection_shard2_replica_n2] Registered new searcher Searcher@56b871e3[authCollection_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 720646 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 720646 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 720664 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 720664 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 720665 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@758c0f1d[authCollection_shard3_replica_n4] main]
   [junit4]   2> 720744 INFO  (searcherExecutor-1450-thread-1-processing-n:127.0.0.1:42743_solr x:authCollection_shard1_replica_n1 c:authCollection s:shard1 r:core_node3) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.SolrCore [authCollection_shard1_replica_n1] Registered new searcher Searcher@584587f7[authCollection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 720777 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 720778 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 720779 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 720779 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1630296332811894784
   [junit4]   2> 720799 INFO  (searcherExecutor-1452-thread-1-processing-n:127.0.0.1:38331_solr x:authCollection_shard3_replica_n4 c:authCollection s:shard3 r:core_node6) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SolrCore [authCollection_shard3_replica_n4] Registered new searcher Searcher@758c0f1d[authCollection_shard3_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 720800 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.ZkShardTerms Successful update of terms at /collections/authCollection/terms/shard2 to Terms{values={core_node5=0}, version=0}
   [junit4]   2> 720800 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/authCollection/leaders/shard2
   [junit4]   2> 720823 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.ZkShardTerms Successful update of terms at /collections/authCollection/terms/shard3 to Terms{values={core_node6=0}, version=0}
   [junit4]   2> 720823 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/authCollection/leaders/shard3
   [junit4]   2> 720825 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 720842 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 720860 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/authCollection/terms/shard1 to Terms{values={core_node3=0}, version=0}
   [junit4]   2> 720860 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:38331/solr/authCollection_shard3_replica_n4/
   [junit4]   2> 720860 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/authCollection/leaders/shard1
   [junit4]   2> 720860 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 720860 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SyncStrategy https://127.0.0.1:38331/solr/authCollection_shard3_replica_n4/ has no replicas
   [junit4]   2> 720860 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/authCollection/leaders/shard3/leader after winning as /collections/authCollection/leader_elect/shard3/election/74489578049110025-core_node6-n_0000000000
   [junit4]   2> 720879 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:38331/solr/authCollection_shard3_replica_n4/ shard3
   [junit4]   2> 720880 INFO  (zkCallback-2077-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 720880 INFO  (zkCallback-2077-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 720932 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 720954 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=conf&newCollection=true&name=authCollection_shard3_replica_n4&action=CREATE&numShards=3&collection=authCollection&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2718
   [junit4]   2> 720970 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 720970 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 720970 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:44906/solr/authCollection_shard2_replica_n2/
   [junit4]   2> 720970 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 720970 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.SyncStrategy https://127.0.0.1:44906/solr/authCollection_shard2_replica_n2/ has no replicas
   [junit4]   2> 720970 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/authCollection/leaders/shard2/leader after winning as /collections/authCollection/leader_elect/shard2/election/74489578049110029-core_node5-n_0000000000
   [junit4]   2> 720988 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 720988 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 720988 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:42743/solr/authCollection_shard1_replica_n1/
   [junit4]   2> 720988 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 720988 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:42743/solr/authCollection_shard1_replica_n1/ has no replicas
   [junit4]   2> 720988 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/authCollection/leaders/shard1/leader after winning as /collections/authCollection/leader_elect/shard1/election/74489578049110023-core_node3-n_0000000000
   [junit4]   2> 720990 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:42743/solr/authCollection_shard1_replica_n1/ shard1
   [junit4]   2> 721005 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:44906/solr/authCollection_shard2_replica_n2/ shard2
   [junit4]   2> 721151 INFO  (zkCallback-2099-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 721151 INFO  (zkCallback-2067-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 721152 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 721152 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 721152 INFO  (zkCallback-2077-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 721153 INFO  (zkCallback-2077-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 721166 INFO  (zkCallback-2077-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 721171 INFO  (qtp128994539-5503) [n:127.0.0.1:42743_solr c:authCollection s:shard1 r:core_node3 x:authCollection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=authCollection_shard1_replica_n1&action=CREATE&numShards=3&collection=authCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3157
   [junit4]   2> 721210 INFO  (qtp1743232006-5491) [n:127.0.0.1:44906_solr c:authCollection s:shard2 r:core_node5 x:authCollection_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=authCollection_shard2_replica_n2&action=CREATE&numShards=3&collection=authCollection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3100
   [junit4]   2> 721276 INFO  (qtp128994539-5505) [n:127.0.0.1:42743_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 721333 INFO  (zkCallback-2099-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 721333 INFO  (zkCallback-2067-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 721334 INFO  (qtp128994539-5505) [n:127.0.0.1:42743_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf&name=authCollection&nrtReplicas=1&action=CREATE&numShards=3&wt=javabin&version=2} status=0 QTime=4319
   [junit4]   2> 721348 INFO  (zkCallback-2077-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 721364 INFO  (zkCallback-2077-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 721365 INFO  (zkCallback-2099-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 721368 INFO  (zkCallback-2077-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 721401 INFO  (OverseerCollectionConfigSetProcessor-74489578049110025-127.0.0.1:38331_solr-n_0000000000) [n:127.0.0.1:38331_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 721457 INFO  (zkCallback-2067-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 721572 INFO  (qtp2051655855-5510) [n:127.0.0.1:38331_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/authentication params={} status=0 QTime=0
   [junit4]   2> 721574 INFO  (qtp2051655855-5511) [n:127.0.0.1:38331_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/authentication params={} status=0 QTime=0
   [junit4]   2> 721603 INFO  (zkCallback-2077-thread-4) [    ] o.a.s.c.CoreContainer Security node changed, reloading security.json
   [junit4]   2> 721603 INFO  (zkCallback-2077-thread-4) [    ] o.a.s.c.CoreContainer Initializing authorization plugin: solr.RuleBasedAuthorizationPlugin
   [junit4]   2> 721620 INFO  (zkCallback-2099-thread-2) [    ] o.a.s.c.CoreContainer Security node changed, reloading security.json
   [junit4]   2> 721620 INFO  (zkCallback-2099-thread-2) [    ] o.a.s.c.CoreContainer Initializing authorization plugin: solr.RuleBasedAuthorizationPlugin
   [junit4]   2> 721621 INFO  (zkCallback-2067-thread-2) [    ] o.a.s.c.CoreContainer Security node changed, reloading security.json
   [junit4]   2> 721621 INFO  (zkCallback-2067-thread-2) [    ] o.a.s.c.CoreContainer Initializing authorization plugin: solr.RuleBasedAuthorizationPlugin
   [junit4]   2> 721625 INFO  (qtp2051655855-5512) [n:127.0.0.1:38331_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/authentication params={} status=0 QTime=0
   [junit4]   2> 721625 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1857681463
   [junit4]   2> 721625 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:38331_solr
   [junit4]   2> 721644 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.c.CoreContainer Publish this node as DOWN...
   [junit4]   2> 721644 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.c.ZkController Publish node=127.0.0.1:38331_solr as DOWN
   [junit4]   2> 721767 INFO  (coreCloseExecutor-1465-thread-1) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SolrCore [authCollection_shard3_replica_n4]  CLOSING SolrCore org.apache.solr.core.SolrCore@54db9bf7
   [junit4]   2> 721767 INFO  (coreCloseExecutor-1465-thread-1) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.authCollection.shard3.replica_n4, tag=54db9bf7
   [junit4]   2> 721767 INFO  (coreCloseExecutor-1465-thread-1) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@48d66c41: rootName = solr_38331, domain = solr.core.authCollection.shard3.replica_n4, service url = null, agent id = null] for registry solr.core.authCollection.shard3.replica_n4 / com.codahale.metrics.MetricRegistry@50e75482
   [junit4]   2> 721846 INFO  (coreCloseExecutor-1465-thread-1) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.authCollection.shard3.leader, tag=54db9bf7
   [junit4]   2> 721935 INFO  (zkCallback-2099-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 721968 INFO  (zkCallback-2077-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 721968 INFO  (zkCallback-2077-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 721989 INFO  (zkCallback-2077-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 721989 INFO  (zkCallback-2067-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 721990 INFO  (zkCallback-2099-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 722008 INFO  (zkCallback-2067-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 722028 INFO  (zkCallback-2099-thread-2) [    ] o.a.s.c.CoreContainer Initializing authentication plugin: solr.BasicAuthPlugin
   [junit4]   2> 722047 INFO  (zkCallback-2067-thread-2) [    ] o.a.s.c.CoreContainer Initializing authentication plugin: solr.BasicAuthPlugin
   [junit4]   2> 722064 INFO  (zkCallback-2077-thread-4) [    ] o.a.s.c.CoreContainer Initializing authentication plugin: solr.BasicAuthPlugin
   [junit4]   2> 722146 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 722146 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@57a92f4a: rootName = solr_38331, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@15306b67
   [junit4]   2> 722167 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 722167 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6871d033: rootName = solr_38331, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@4fd396be
   [junit4]   2> 722169 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 722169 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@610ba4ed: rootName = solr_38331, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@78e3fbb5
   [junit4]   2> 722169 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 722257 INFO  (closeThreadPool-2119-thread-2) [    ] o.a.s.c.Overseer Overseer (id=74489578049110025-127.0.0.1:38331_solr-n_0000000000) closing
   [junit4]   2> 722442 INFO  (OverseerAutoScalingTriggerThread-74489578049110025-127.0.0.1:38331_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 722442 INFO  (OverseerStateUpdate-74489578049110025-127.0.0.1:38331_solr-n_0000000000) [n:127.0.0.1:38331_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:38331_solr
   [junit4]   2> 722512 INFO  (closeThreadPool-2119-thread-1) [    ] o.a.s.c.Overseer Overseer (id=74489578049110025-127.0.0.1:38331_solr-n_0000000000) closing
   [junit4]   2> 722866 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.c.Overseer Overseer (id=74489578049110025-127.0.0.1:38331_solr-n_0000000000) closing
   [junit4]   2> 722866 INFO  (zkCallback-2106-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 722867 INFO  (zkCallback-2089-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 722867 INFO  (zkCallback-2099-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 722867 INFO  (zkCallback-2112-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 722867 INFO  (zkCallback-2067-thread-1) [    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:42743_solr
   [junit4]   2> 722867 INFO  (zkCallback-2067-thread-3) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 722900 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@363c9538{SSL,[ssl, http/1.1]}{127.0.0.1:0}
   [junit4]   2> 722901 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@61926aeb{/solr,null,UNAVAILABLE}
   [junit4]   2> 722901 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 722902 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 722903 WARN  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 722903 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 38331 try number 1 ...
   [junit4]   2> 722903 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 722936 INFO  (zkCallback-2067-thread-1) [n:127.0.0.1:42743_solr    ] o.a.s.c.Overseer Overseer (id=74489578049110023-127.0.0.1:42743_solr-n_0000000001) starting
   [junit4]   2> 722964 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 722964 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 722965 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 722965 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1a1698{/solr,null,AVAILABLE}
   [junit4]   2> 722965 WARN  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@5bc8e331[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 722993 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@161e780{SSL,[ssl, http/1.1]}{127.0.0.1:38331}
   [junit4]   2> 722993 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.e.j.s.Server Started @723090ms
   [junit4]   2> 722993 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38331}
   [junit4]   2> 723037 INFO  (OverseerStateUpdate-74489578049110023-127.0.0.1:42743_solr-n_0000000001) [n:127.0.0.1:42743_solr    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:42743_solr
   [junit4]   2> 723046 ERROR (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 723046 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 723046 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.1.0
   [junit4]   2> 723046 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 723046 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 723046 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-04-09T01:02:23.001Z
   [junit4]   2> 723082 INFO  (zkConnectionManagerCallback-2121-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 723094 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 725415 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 725416 WARN  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 725472 WARN  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@49240596[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 725489 WARN  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 725490 WARN  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@1ffddfba[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 725490 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45637/solr
   [junit4]   2> 725513 INFO  (zkConnectionManagerCallback-2128-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 725515 INFO  (zkConnectionManagerCallback-2130-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 725535 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [n:127.0.0.1:38331_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 725679 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [n:127.0.0.1:38331_solr    ] o.a.s.c.ZkController Publish node=127.0.0.1:38331_solr as DOWN
   [junit4]   2> 725680 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [n:127.0.0.1:38331_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 725680 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [n:127.0.0.1:38331_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38331_solr
   [junit4]   2> 725681 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [n:127.0.0.1:38331_solr    ] o.a.s.c.CoreContainer Initializing authorization plugin: solr.RuleBasedAuthorizationPlugin
   [junit4]   2> 725681 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [n:127.0.0.1:38331_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: solr.BasicAuthPlugin
   [junit4]   2> 725695 INFO  (zkCallback-2112-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 725695 INFO  (zkCallback-2067-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 725695 INFO  (zkCallback-2099-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 725695 INFO  (zkCallback-2106-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 725695 INFO  (zkCallback-2067-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 725695 INFO  (zkCallback-2067-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 725696 INFO  (zkCallback-2099-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 725715 INFO  (zkCallback-2099-thread-3) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/authCollection/state.json] for collection [authCollection] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 725715 INFO  (zkCallback-2129-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 725739 INFO  (zkCallback-2089-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 726227 INFO  (zkConnectionManagerCallback-2137-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 726228 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [n:127.0.0.1:38331_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 726229 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [n:127.0.0.1:38331_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45637/solr ready
   [junit4]   2> 726242 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [n:127.0.0.1:38331_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 726424 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [n:127.0.0.1:38331_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38331.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@216a5b5
   [junit4]   2> 726461 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [n:127.0.0.1:38331_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38331.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@216a5b5
   [junit4]   2> 726461 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [n:127.0.0.1:38331_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38331.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@216a5b5
   [junit4]   2> 726463 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [n:127.0.0.1:38331_solr    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_BE7C305C7E19584C-001/tempDir-001/node3/.
   [junit4]   2> 726463 INFO  (TEST-BasicAuthIntegrationTest.testBasicAuth-seed#[BE7C305C7E19584C]) [n:127.0.0.1:38331_solr    ] o.a.s.c.CorePropertiesLocator Cores are: [authCollection_shard3_replica_n4]
   [junit4]   2> 726650 INFO  (coreLoadExecutor-1479-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.1.0
   [junit4]   2> 726850 INFO  (coreLoadExecutor-1479-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr    ] o.a.s.s.IndexSchema [authCollection_shard3_replica_n4] Schema name=minimal
   [junit4]   2> 726852 INFO  (coreLoadExecutor-1479-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 726920 INFO  (coreLoadExecutor-1479-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.1.0
   [junit4]   2> 727498 INFO  (coreLoadExecutor-1479-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.s.IndexSchema [authCollection_shard3_replica_n4] Schema name=minimal
   [junit4]   2> 727500 INFO  (coreLoadExecutor-1479-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 727500 INFO  (coreLoadExecutor-1479-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'authCollection_shard3_replica_n4' using configuration from collection authCollection, trusted=true
   [junit4]   2> 727500 INFO  (coreLoadExecutor-1479-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38331.solr.core.authCollection.shard3.replica_n4' (registry 'solr.core.authCollection.shard3.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@216a5b5
   [junit4]   2> 727627 INFO  (coreLoadExecutor-1479-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SolrCore [[authCollection_shard3_replica_n4] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_BE7C305C7E19584C-001/tempDir-001/node3/authCollection_shard3_replica_n4], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_BE7C305C7E19584C-001/tempDir-001/node3/./authCollection_shard3_replica_n4/data/]
   [junit4]   2> 728336 INFO  (coreLoadExecutor-1479-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 728336 INFO  (coreLoadExecutor-1479-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 728338 INFO  (coreLoadExecutor-1479-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 728338 INFO  (coreLoadExecutor-1479-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 728339 INFO  (coreLoadExecutor-1479-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@1f6423f[authCollection_shard3_replica_n4] main]
   [junit4]   2> 728352 INFO  (coreLoadExecutor-1479-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 728352 INFO  (coreLoadExecutor-1479-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 728352 INFO  (coreLoadExecutor-1479-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 728353 INFO  (coreLoadExecutor-1479-thread-1-processing-n:127.0.0.1:38331_solr) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1630296340753809408
   [junit4]   2> 728411 INFO  (coreZkRegister-1471-thread-1-processing-n:127.0.0.1:38331_solr x:authCollection_shard3_replica_n4 c:authCollection s:shard3 r:core_node6) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/authCollection/leaders/shard3
   [junit4]   2> 728412 INFO  (coreZkRegister-1471-thread-1-processing-n:127.0.0.1:38331_solr x:authCollection_shard3_replica_n4 c:authCollection s:shard3 r:core_node6) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 728412 INFO  (coreZkRegister-1471-thread-1-processing-n:127.0.0.1:38331_solr x:authCollection_shard3_replica_n4 c:authCollection s:shard3 r:core_node6) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 728412 INFO  (coreZkRegister-1471-thread-1-processing-n:127.0.0.1:38331_solr x:authCollection_shard3_replica_n4 c:authCollection s:shard3 r:core_node6) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:38331/solr/authCollection_shard3_replica_n4/
   [junit4]   2> 728445 INFO  (searcherExecutor-1480-thread-1-processing-n:127.0.0.1:38331_solr x:authCollection_shard3_replica_n4 c:authCollection s:shard3 r:core_node6) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SolrCore [authCollection_shard3_replica_n4] Registered new searcher Searcher@1f6423f[authCollection_shard3_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 728445 INFO  (coreZkRegister-1471-thread-1-processing-n:127.0.0.1:38331_solr x:authCollection_shard3_replica_n4 c:authCollection s:shard3 r:core_node6) [n:127.0.0.1:38331_solr c:authCollection s:shard3 r:core_node6 x:authCollection_shard3_replica_n4] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 728446 INFO  (coreZkRegister-14

[...truncated too long message...]

ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/null1707949454
     [copy] Copying 240 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/null1707949454
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/null1707949454

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] 	confs: [default]
[ivy:cachepath] 	found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] 	found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] 	found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] 	found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] 	found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] 	found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 54ms :: artifacts dl 1ms
	---------------------------------------------------------------------
	|                  |            modules            ||   artifacts   |
	|       conf       | number| search|dwnlded|evicted|| number|dwnlded|
	---------------------------------------------------------------------
	|      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
	---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 233 minutes 19 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2847)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2726)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2707)
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3086)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.FilePath.act(FilePath.java:1072)
		at hudson.FilePath.act(FilePath.java:1061)
		at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
		at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
		at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
		at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
		at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
		at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
		at hudson.model.Build$BuildExecution.post2(Build.java:186)
		at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
		at hudson.model.Run.execute(Run.java:1835)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3088)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath.act(FilePath.java:1074)
	at hudson.FilePath.act(FilePath.java:1061)
	at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
	at hudson.model.Build$BuildExecution.post2(Build.java:186)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
	at hudson.model.Run.execute(Run.java:1835)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)

[JENKINS] Lucene-Solr-Tests-8.x - Build # 109 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-8.x/109/

5 tests failed.
FAILED:  org.apache.lucene.classification.utils.ConfusionMatrixGeneratorTest.testGetConfusionMatrixWithFLTKNN

Error Message:
expected:<7> but was:<6>

Stack Trace:
java.lang.AssertionError: expected:<7> but was:<6>
	at __randomizedtesting.SeedInfo.seed([FF766BBFDB3E9F17:4AB8A88D43F24098]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.failNotEquals(Assert.java:834)
	at org.junit.Assert.assertEquals(Assert.java:645)
	at org.junit.Assert.assertEquals(Assert.java:631)
	at org.apache.lucene.classification.utils.ConfusionMatrixGeneratorTest.checkCM(ConfusionMatrixGeneratorTest.java:110)
	at org.apache.lucene.classification.utils.ConfusionMatrixGeneratorTest.testGetConfusionMatrixWithFLTKNN(ConfusionMatrixGeneratorTest.java:186)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.analytics.legacy.LegacyNoFacetCloudTest.sumTest

Error Message:
Error starting up MiniSolrCloudCluster

Stack Trace:
java.lang.Exception: Error starting up MiniSolrCloudCluster
	at org.apache.solr.cloud.MiniSolrCloudCluster.checkForExceptions(MiniSolrCloudCluster.java:652)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:306)
	at org.apache.solr.cloud.SolrCloudTestCase$Builder.build(SolrCloudTestCase.java:212)
	at org.apache.solr.cloud.SolrCloudTestCase$Builder.configure(SolrCloudTestCase.java:204)
	at org.apache.solr.analytics.legacy.LegacyAbstractAnalyticsCloudTest.setupCollection(LegacyAbstractAnalyticsCloudTest.java:49)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:972)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: java.lang.RuntimeException: Jetty/Solr unresponsive
		at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:507)
		at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:465)
		at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:467)
		at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:300)
		at java.util.concurrent.FutureTask.run(FutureTask.java:266)
		at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
		... 1 more


FAILED:  junit.framework.TestSuite.org.apache.solr.analytics.legacy.LegacyNoFacetCloudTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.analytics.legacy.LegacyNoFacetCloudTest:     1) Thread[id=2248, name=Scheduler-851174062, state=TIMED_WAITING, group=TGRP-LegacyNoFacetCloudTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=2243, name=qtp1404790099-2243, state=TIMED_WAITING, group=TGRP-LegacyNoFacetCloudTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=2240, name=qtp1404790099-2240, state=RUNNABLE, group=TGRP-LegacyNoFacetCloudTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1984061638.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=2247, name=qtp1404790099-2247, state=TIMED_WAITING, group=TGRP-LegacyNoFacetCloudTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=2246, name=qtp1404790099-2246, state=TIMED_WAITING, group=TGRP-LegacyNoFacetCloudTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=2321, name=zkConnectionManagerCallback-1096-thread-1, state=WAITING, group=TGRP-LegacyNoFacetCloudTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=2244, name=qtp1404790099-2244, state=TIMED_WAITING, group=TGRP-LegacyNoFacetCloudTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=2241, name=qtp1404790099-2241, state=RUNNABLE, group=TGRP-LegacyNoFacetCloudTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1984061638.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=2242, name=qtp1404790099-2242-acceptor-0@408b441-ServerConnector@acf5ab0{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:39310}, state=RUNNABLE, group=TGRP-LegacyNoFacetCloudTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=2245, name=qtp1404790099-2245, state=TIMED_WAITING, group=TGRP-LegacyNoFacetCloudTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.analytics.legacy.LegacyNoFacetCloudTest: 
   1) Thread[id=2248, name=Scheduler-851174062, state=TIMED_WAITING, group=TGRP-LegacyNoFacetCloudTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=2243, name=qtp1404790099-2243, state=TIMED_WAITING, group=TGRP-LegacyNoFacetCloudTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=2240, name=qtp1404790099-2240, state=RUNNABLE, group=TGRP-LegacyNoFacetCloudTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1984061638.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=2247, name=qtp1404790099-2247, state=TIMED_WAITING, group=TGRP-LegacyNoFacetCloudTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=2246, name=qtp1404790099-2246, state=TIMED_WAITING, group=TGRP-LegacyNoFacetCloudTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=2321, name=zkConnectionManagerCallback-1096-thread-1, state=WAITING, group=TGRP-LegacyNoFacetCloudTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=2244, name=qtp1404790099-2244, state=TIMED_WAITING, group=TGRP-LegacyNoFacetCloudTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=2241, name=qtp1404790099-2241, state=RUNNABLE, group=TGRP-LegacyNoFacetCloudTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1984061638.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=2242, name=qtp1404790099-2242-acceptor-0@408b441-ServerConnector@acf5ab0{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:39310}, state=RUNNABLE, group=TGRP-LegacyNoFacetCloudTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=2245, name=qtp1404790099-2245, state=TIMED_WAITING, group=TGRP-LegacyNoFacetCloudTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:656)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:46)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:720)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([70F60D4585362678]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.analytics.legacy.LegacyNoFacetCloudTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=2248, name=Scheduler-851174062, state=TIMED_WAITING, group=TGRP-LegacyNoFacetCloudTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=2240, name=qtp1404790099-2240, state=RUNNABLE, group=TGRP-LegacyNoFacetCloudTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1984061638.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=2321, name=zkConnectionManagerCallback-1096-thread-1, state=WAITING, group=TGRP-LegacyNoFacetCloudTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=2241, name=qtp1404790099-2241, state=RUNNABLE, group=TGRP-LegacyNoFacetCloudTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)         at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)         at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1984061638.run(Unknown Source)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=2248, name=Scheduler-851174062, state=TIMED_WAITING, group=TGRP-LegacyNoFacetCloudTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=2240, name=qtp1404790099-2240, state=RUNNABLE, group=TGRP-LegacyNoFacetCloudTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1984061638.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=2321, name=zkConnectionManagerCallback-1096-thread-1, state=WAITING, group=TGRP-LegacyNoFacetCloudTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=2241, name=qtp1404790099-2241, state=RUNNABLE, group=TGRP-LegacyNoFacetCloudTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:423)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:360)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
        at org.eclipse.jetty.io.ManagedSelector$$Lambda$61/1984061638.run(Unknown Source)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([70F60D4585362678]:0)


FAILED:  org.apache.solr.cloud.DeleteReplicaTest.deleteLiveReplicaTest

Error Message:
expected:<0> but was:<1>

Stack Trace:
java.lang.AssertionError: expected:<0> but was:<1>
	at __randomizedtesting.SeedInfo.seed([C06A63DF0E509539:6D0AD7D4136F3D4C]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.failNotEquals(Assert.java:834)
	at org.junit.Assert.assertEquals(Assert.java:645)
	at org.junit.Assert.assertEquals(Assert.java:631)
	at org.apache.solr.cloud.DeleteReplicaTest.deleteLiveReplicaTest(DeleteReplicaTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 7072 lines...]
   [junit4] Suite: org.apache.lucene.classification.utils.ConfusionMatrixGeneratorTest
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ConfusionMatrixGeneratorTest -Dtests.method=testGetConfusionMatrixWithFLTKNN -Dtests.seed=FF766BBFDB3E9F17 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=el-GR -Dtests.timezone=Etc/Greenwich -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 10.8s J2 | ConfusionMatrixGeneratorTest.testGetConfusionMatrixWithFLTKNN <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<7> but was:<6>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([FF766BBFDB3E9F17:4AB8A88D43F24098]:0)
   [junit4]    > 	at org.apache.lucene.classification.utils.ConfusionMatrixGeneratorTest.checkCM(ConfusionMatrixGeneratorTest.java:110)
   [junit4]    > 	at org.apache.lucene.classification.utils.ConfusionMatrixGeneratorTest.testGetConfusionMatrixWithFLTKNN(ConfusionMatrixGeneratorTest.java:186)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene80): {bool=PostingsFormat(name=Direct), cat=PostingsFormat(name=Direct), text=Lucene50(blocksize=128)}, docValues:{}, maxPointsInLeafNode=602, maxMBSortInHeap=7.150728974079945, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@78347bd), locale=el-GR, timezone=Etc/Greenwich
   [junit4]   2> NOTE: Linux 4.4.0-112-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=261820528,total=333971456
   [junit4]   2> NOTE: All tests run in this JVM: [CachingNaiveBayesClassifierTest, ConfusionMatrixGeneratorTest]
   [junit4] Completed [6/11 (1!)] on J2 in 18.13s, 7 tests, 1 failure <<< FAILURES!

[...truncated 7265 lines...]
   [junit4] Suite: org.apache.solr.cloud.DeleteReplicaTest
   [junit4]   2> 3066863 INFO  (SUITE-DeleteReplicaTest-seed#[C06A63DF0E509539]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteReplicaTest_C06A63DF0E509539-001/init-core-data-001
   [junit4]   2> 3066876 WARN  (SUITE-DeleteReplicaTest-seed#[C06A63DF0E509539]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
   [junit4]   2> 3066876 INFO  (SUITE-DeleteReplicaTest-seed#[C06A63DF0E509539]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 3066877 INFO  (SUITE-DeleteReplicaTest-seed#[C06A63DF0E509539]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 3066896 INFO  (TEST-DeleteReplicaTest.deleteReplicaByCount-seed#[C06A63DF0E509539]) [    ] o.a.s.SolrTestCaseJ4 ###Starting deleteReplicaByCount
   [junit4]   2> 3066897 INFO  (TEST-DeleteReplicaTest.deleteReplicaByCount-seed#[C06A63DF0E509539]) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteReplicaTest_C06A63DF0E509539-001/tempDir-001
   [junit4]   2> 3066897 INFO  (TEST-DeleteReplicaTest.deleteReplicaByCount-seed#[C06A63DF0E509539]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 3066900 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 3066900 INFO  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 3067001 INFO  (TEST-DeleteReplicaTest.deleteReplicaByCount-seed#[C06A63DF0E509539]) [    ] o.a.s.c.ZkTestServer start zk server on port:39162
   [junit4]   2> 3067001 INFO  (TEST-DeleteReplicaTest.deleteReplicaByCount-seed#[C06A63DF0E509539]) [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:39162
   [junit4]   2> 3067001 INFO  (TEST-DeleteReplicaTest.deleteReplicaByCount-seed#[C06A63DF0E509539]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 39162
   [junit4]   2> 3067005 INFO  (zkConnectionManagerCallback-9758-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3067074 INFO  (zkConnectionManagerCallback-9760-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3067076 INFO  (zkConnectionManagerCallback-9762-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3067077 WARN  (jetty-launcher-9763-thread-1) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 3067077 INFO  (jetty-launcher-9763-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 3067077 INFO  (jetty-launcher-9763-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 3067078 INFO  (jetty-launcher-9763-thread-1) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 3067095 WARN  (jetty-launcher-9763-thread-3) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 3067095 INFO  (jetty-launcher-9763-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 3067095 INFO  (jetty-launcher-9763-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 3067095 INFO  (jetty-launcher-9763-thread-3) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 3067095 WARN  (jetty-launcher-9763-thread-4) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 3067096 INFO  (jetty-launcher-9763-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 3067096 INFO  (jetty-launcher-9763-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 3067096 INFO  (jetty-launcher-9763-thread-4) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 3067111 WARN  (jetty-launcher-9763-thread-2) [    ] o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 3067111 INFO  (jetty-launcher-9763-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 3067111 INFO  (jetty-launcher-9763-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 3067111 INFO  (jetty-launcher-9763-thread-2) [    ] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 3067139 INFO  (jetty-launcher-9763-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3067139 INFO  (jetty-launcher-9763-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3067139 INFO  (jetty-launcher-9763-thread-1) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 3067139 INFO  (jetty-launcher-9763-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@75043b4b{/solr,null,AVAILABLE}
   [junit4]   2> 3067195 INFO  (jetty-launcher-9763-thread-3) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3067195 INFO  (jetty-launcher-9763-thread-3) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3067195 INFO  (jetty-launcher-9763-thread-3) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 3067252 INFO  (jetty-launcher-9763-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3067252 INFO  (jetty-launcher-9763-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3067252 INFO  (jetty-launcher-9763-thread-2) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 3067269 INFO  (jetty-launcher-9763-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@23f1266d{/solr,null,AVAILABLE}
   [junit4]   2> 3067269 INFO  (jetty-launcher-9763-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@54d3389{/solr,null,AVAILABLE}
   [junit4]   2> 3067285 INFO  (jetty-launcher-9763-thread-4) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 3067285 INFO  (jetty-launcher-9763-thread-4) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 3067285 INFO  (jetty-launcher-9763-thread-4) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 3067285 INFO  (jetty-launcher-9763-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7aab2fbf{/solr,null,AVAILABLE}
   [junit4]   2> 3067286 INFO  (jetty-launcher-9763-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@40ff732d{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:33253}
   [junit4]   2> 3067342 INFO  (jetty-launcher-9763-thread-1) [    ] o.e.j.s.Server Started @3067398ms
   [junit4]   2> 3067342 INFO  (jetty-launcher-9763-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33253}
   [junit4]   2> 3067342 ERROR (jetty-launcher-9763-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3067342 INFO  (jetty-launcher-9763-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3067342 INFO  (jetty-launcher-9763-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.1.0
   [junit4]   2> 3067342 INFO  (jetty-launcher-9763-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3067342 INFO  (jetty-launcher-9763-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3067342 INFO  (jetty-launcher-9763-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-04-08T15:36:44.174Z
   [junit4]   2> 3067378 INFO  (jetty-launcher-9763-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@17c8d466{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:38261}
   [junit4]   2> 3067378 INFO  (jetty-launcher-9763-thread-4) [    ] o.e.j.s.Server Started @3067434ms
   [junit4]   2> 3067378 INFO  (jetty-launcher-9763-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@9900306{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:40248}
   [junit4]   2> 3067378 INFO  (jetty-launcher-9763-thread-2) [    ] o.e.j.s.Server Started @3067434ms
   [junit4]   2> 3067378 INFO  (jetty-launcher-9763-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38261}
   [junit4]   2> 3067378 INFO  (jetty-launcher-9763-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40248}
   [junit4]   2> 3067378 INFO  (zkConnectionManagerCallback-9765-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3067378 ERROR (jetty-launcher-9763-thread-4) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3067378 INFO  (jetty-launcher-9763-thread-4) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3067378 ERROR (jetty-launcher-9763-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3067378 INFO  (jetty-launcher-9763-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3067378 INFO  (jetty-launcher-9763-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.1.0
   [junit4]   2> 3067378 INFO  (jetty-launcher-9763-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.1.0
   [junit4]   2> 3067378 INFO  (jetty-launcher-9763-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3067378 INFO  (jetty-launcher-9763-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3067378 INFO  (jetty-launcher-9763-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3067378 INFO  (jetty-launcher-9763-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3067378 INFO  (jetty-launcher-9763-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-04-08T15:36:44.210Z
   [junit4]   2> 3067378 INFO  (jetty-launcher-9763-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-04-08T15:36:44.210Z
   [junit4]   2> 3067379 INFO  (jetty-launcher-9763-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6488dae7{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:43263}
   [junit4]   2> 3067379 INFO  (jetty-launcher-9763-thread-3) [    ] o.e.j.s.Server Started @3067435ms
   [junit4]   2> 3067379 INFO  (jetty-launcher-9763-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43263}
   [junit4]   2> 3067379 ERROR (jetty-launcher-9763-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 3067379 INFO  (jetty-launcher-9763-thread-3) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 3067379 INFO  (jetty-launcher-9763-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.1.0
   [junit4]   2> 3067379 INFO  (jetty-launcher-9763-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 3067379 INFO  (jetty-launcher-9763-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 3067379 INFO  (jetty-launcher-9763-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2019-04-08T15:36:44.211Z
   [junit4]   2> 3067379 INFO  (jetty-launcher-9763-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3067495 INFO  (zkConnectionManagerCallback-9769-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3067496 INFO  (zkConnectionManagerCallback-9771-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3067524 INFO  (zkConnectionManagerCallback-9767-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3067532 INFO  (jetty-launcher-9763-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3067532 INFO  (jetty-launcher-9763-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3067552 INFO  (jetty-launcher-9763-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 3068097 INFO  (jetty-launcher-9763-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 3068116 WARN  (jetty-launcher-9763-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@11a476e2[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3068181 WARN  (jetty-launcher-9763-thread-1) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@618a246c[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3068182 INFO  (jetty-launcher-9763-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39162/solr
   [junit4]   2> 3068205 INFO  (zkConnectionManagerCallback-9781-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3068241 INFO  (zkConnectionManagerCallback-9783-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3068607 INFO  (jetty-launcher-9763-thread-2) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 3068627 WARN  (jetty-launcher-9763-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@47a33a37[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3068663 WARN  (jetty-launcher-9763-thread-2) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@1df37096[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3068664 INFO  (jetty-launcher-9763-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39162/solr
   [junit4]   2> 3068683 INFO  (zkConnectionManagerCallback-9791-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3068729 INFO  (jetty-launcher-9763-thread-3) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 3068756 INFO  (zkConnectionManagerCallback-9793-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3068790 INFO  (jetty-launcher-9763-thread-2) [n:127.0.0.1:40248_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:40248_solr
   [junit4]   2> 3068791 INFO  (jetty-launcher-9763-thread-2) [n:127.0.0.1:40248_solr    ] o.a.s.c.Overseer Overseer (id=74487354652164106-127.0.0.1:40248_solr-n_0000000000) starting
   [junit4]   2> 3068801 WARN  (jetty-launcher-9763-thread-3) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@e53050d[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3068815 WARN  (jetty-launcher-9763-thread-3) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@ee9a9b[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3068815 INFO  (jetty-launcher-9763-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39162/solr
   [junit4]   2> 3068865 INFO  (zkConnectionManagerCallback-9801-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3068995 INFO  (zkConnectionManagerCallback-9806-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3069009 INFO  (jetty-launcher-9763-thread-1) [n:127.0.0.1:33253_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33253_solr
   [junit4]   2> 3069027 INFO  (zkConnectionManagerCallback-9808-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3069027 INFO  (zkCallback-9792-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3069028 INFO  (jetty-launcher-9763-thread-2) [n:127.0.0.1:40248_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3069055 INFO  (zkCallback-9782-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 3069064 INFO  (jetty-launcher-9763-thread-2) [n:127.0.0.1:40248_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39162/solr ready
   [junit4]   2> 3069100 INFO  (OverseerStateUpdate-74487354652164106-127.0.0.1:40248_solr-n_0000000000) [n:127.0.0.1:40248_solr    ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:40248_solr
   [junit4]   2> 3069129 INFO  (jetty-launcher-9763-thread-2) [n:127.0.0.1:40248_solr    ] o.a.s.c.ZkController Publish node=127.0.0.1:40248_solr as DOWN
   [junit4]   2> 3069150 INFO  (jetty-launcher-9763-thread-2) [n:127.0.0.1:40248_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 3069150 INFO  (jetty-launcher-9763-thread-2) [n:127.0.0.1:40248_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40248_solr
   [junit4]   2> 3069151 INFO  (zkCallback-9782-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3069151 INFO  (zkCallback-9792-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3069230 INFO  (zkConnectionManagerCallback-9815-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3069250 INFO  (zkCallback-9805-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 3069279 INFO  (jetty-launcher-9763-thread-1) [n:127.0.0.1:33253_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3069279 INFO  (jetty-launcher-9763-thread-1) [n:127.0.0.1:33253_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39162/solr ready
   [junit4]   2> 3069291 INFO  (jetty-launcher-9763-thread-1) [n:127.0.0.1:33253_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 3069319 INFO  (jetty-launcher-9763-thread-2) [n:127.0.0.1:40248_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 3069349 INFO  (jetty-launcher-9763-thread-4) [    ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 3069401 WARN  (jetty-launcher-9763-thread-4) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@552c1ef2[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3069513 INFO  (jetty-launcher-9763-thread-1) [n:127.0.0.1:33253_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33253.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e998352
   [junit4]   2> 3069530 WARN  (jetty-launcher-9763-thread-4) [    ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for SslContextFactory@1a7f5d82[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 3069531 INFO  (jetty-launcher-9763-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39162/solr
   [junit4]   2> 3069549 INFO  (jetty-launcher-9763-thread-1) [n:127.0.0.1:33253_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33253.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e998352
   [junit4]   2> 3069549 INFO  (jetty-launcher-9763-thread-1) [n:127.0.0.1:33253_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33253.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e998352
   [junit4]   2> 3069558 INFO  (zkConnectionManagerCallback-9823-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3069583 INFO  (jetty-launcher-9763-thread-1) [n:127.0.0.1:33253_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteReplicaTest_C06A63DF0E509539-001/tempDir-001/node1/.
   [junit4]   2> 3069606 INFO  (jetty-launcher-9763-thread-2) [n:127.0.0.1:40248_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40248.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e998352
   [junit4]   2> 3069651 INFO  (jetty-launcher-9763-thread-3) [n:127.0.0.1:43263_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3069695 INFO  (jetty-launcher-9763-thread-2) [n:127.0.0.1:40248_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40248.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e998352
   [junit4]   2> 3069695 INFO  (jetty-launcher-9763-thread-2) [n:127.0.0.1:40248_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40248.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e998352
   [junit4]   2> 3069696 INFO  (zkConnectionManagerCallback-9825-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3069729 INFO  (jetty-launcher-9763-thread-2) [n:127.0.0.1:40248_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteReplicaTest_C06A63DF0E509539-001/tempDir-001/node4/.
   [junit4]   2> 3069781 INFO  (jetty-launcher-9763-thread-3) [n:127.0.0.1:43263_solr    ] o.a.s.c.ZkController Publish node=127.0.0.1:43263_solr as DOWN
   [junit4]   2> 3069782 INFO  (jetty-launcher-9763-thread-4) [n:127.0.0.1:38261_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 3069782 INFO  (jetty-launcher-9763-thread-3) [n:127.0.0.1:43263_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 3069782 INFO  (jetty-launcher-9763-thread-3) [n:127.0.0.1:43263_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43263_solr
   [junit4]   2> 3069784 INFO  (zkCallback-9792-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 3069784 INFO  (zkCallback-9805-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 3069784 INFO  (zkCallback-9782-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 3069802 INFO  (zkCallback-9824-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 3069802 INFO  (zkCallback-9807-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 3069802 INFO  (zkCallback-9814-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 3069803 INFO  (jetty-launcher-9763-thread-4) [n:127.0.0.1:38261_solr    ] o.a.s.c.ZkController Publish node=127.0.0.1:38261_solr as DOWN
   [junit4]   2> 3069804 INFO  (jetty-launcher-9763-thread-4) [n:127.0.0.1:38261_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 3069804 INFO  (jetty-launcher-9763-thread-4) [n:127.0.0.1:38261_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38261_solr
   [junit4]   2> 3069805 INFO  (zkCallback-9805-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 3069805 INFO  (zkCallback-9782-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 3069805 INFO  (zkCallback-9792-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 3069817 INFO  (zkCallback-9807-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 3069818 INFO  (zkCallback-9824-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 3069818 INFO  (zkCallback-9814-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 3070056 INFO  (zkConnectionManagerCallback-9832-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3070069 INFO  (jetty-launcher-9763-thread-3) [n:127.0.0.1:43263_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 3070070 INFO  (jetty-launcher-9763-thread-3) [n:127.0.0.1:43263_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39162/solr ready
   [junit4]   2> 3070071 INFO  (jetty-launcher-9763-thread-3) [n:127.0.0.1:43263_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 3070089 INFO  (zkConnectionManagerCallback-9837-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3070132 INFO  (jetty-launcher-9763-thread-3) [n:127.0.0.1:43263_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43263.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e998352
   [junit4]   2> 3070169 INFO  (jetty-launcher-9763-thread-3) [n:127.0.0.1:43263_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43263.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e998352
   [junit4]   2> 3070169 INFO  (jetty-launcher-9763-thread-3) [n:127.0.0.1:43263_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43263.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e998352
   [junit4]   2> 3070182 INFO  (jetty-launcher-9763-thread-3) [n:127.0.0.1:43263_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteReplicaTest_C06A63DF0E509539-001/tempDir-001/node2/.
   [junit4]   2> 3070198 INFO  (jetty-launcher-9763-thread-4) [n:127.0.0.1:38261_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 3070380 INFO  (jetty-launcher-9763-thread-4) [n:127.0.0.1:38261_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39162/solr ready
   [junit4]   2> 3070393 INFO  (jetty-launcher-9763-thread-4) [n:127.0.0.1:38261_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 3070439 INFO  (jetty-launcher-9763-thread-4) [n:127.0.0.1:38261_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38261.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e998352
   [junit4]   2> 3070476 INFO  (jetty-launcher-9763-thread-4) [n:127.0.0.1:38261_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38261.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e998352
   [junit4]   2> 3070476 INFO  (jetty-launcher-9763-thread-4) [n:127.0.0.1:38261_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38261.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e998352
   [junit4]   2> 3070477 INFO  (jetty-launcher-9763-thread-4) [n:127.0.0.1:38261_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteReplicaTest_C06A63DF0E509539-001/tempDir-001/node3/.
   [junit4]   2> 3070722 INFO  (zkConnectionManagerCallback-9844-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 3070723 INFO  (TEST-DeleteReplicaTest.deleteReplicaByCount-seed#[C06A63DF0E509539]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 3070724 INFO  (TEST-DeleteReplicaTest.deleteReplicaByCount-seed#[C06A63DF0E509539]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39162/solr ready
   [junit4]   2> 3070782 INFO  (qtp1756050340-25918) [n:127.0.0.1:33253_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf&name=deleteByCount&nrtReplicas=3&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 3070888 INFO  (OverseerThreadFactory-7682-thread-1-processing-n:127.0.0.1:40248_solr) [n:127.0.0.1:40248_solr    ] o.a.s.c.a.c.CreateCollectionCmd Create collection deleteByCount
   [junit4]   2> 3071004 INFO  (OverseerStateUpdate-74487354652164106-127.0.0.1:40248_solr-n_0000000000) [n:127.0.0.1:40248_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"deleteByCount",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"deleteByCount_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:33253/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 3071031 INFO  (OverseerStateUpdate-74487354652164106-127.0.0.1:40248_solr-n_0000000000) [n:127.0.0.1:40248_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"deleteByCount",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"deleteByCount_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:38261/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 3071032 INFO  (OverseerStateUpdate-74487354652164106-127.0.0.1:40248_solr-n_0000000000) [n:127.0.0.1:40248_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"deleteByCount",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"deleteByCount_shard1_replica_n3",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:40248/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 3071218 INFO  (qtp1756050340-25920) [n:127.0.0.1:33253_solr    x:deleteByCount_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=deleteByCount_shard1_replica_n1&action=CREATE&numShards=1&collection=deleteByCount&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 3071218 INFO  (qtp1756050340-25920) [n:127.0.0.1:33253_solr    x:deleteByCount_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 3071242 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr    x:deleteByCount_shard1_replica_n3] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node6&collection.configName=conf&newCollection=true&name=deleteByCount_shard1_replica_n3&action=CREATE&numShards=1&collection=deleteByCount&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 3071243 INFO  (qtp524008679-25945) [n:127.0.0.1:38261_solr    x:deleteByCount_shard1_replica_n2] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=deleteByCount_shard1_replica_n2&action=CREATE&numShards=1&collection=deleteByCount&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 3072305 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.1.0
   [junit4]   2> 3072523 INFO  (qtp1756050340-25920) [n:127.0.0.1:33253_solr c:deleteByCount s:shard1 r:core_node4 x:deleteByCount_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.1.0
   [junit4]   2> 3072542 INFO  (qtp524008679-25945) [n:127.0.0.1:38261_solr c:deleteByCount s:shard1 r:core_node5 x:deleteByCount_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.1.0
   [junit4]   2> 3072578 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.s.IndexSchema [deleteByCount_shard1_replica_n3] Schema name=minimal
   [junit4]   2> 3072607 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3072607 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.c.CoreContainer Creating SolrCore 'deleteByCount_shard1_replica_n3' using configuration from collection deleteByCount, trusted=true
   [junit4]   2> 3072607 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40248.solr.core.deleteByCount.shard1.replica_n3' (registry 'solr.core.deleteByCount.shard1.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e998352
   [junit4]   2> 3072608 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.c.SolrCore [[deleteByCount_shard1_replica_n3] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteReplicaTest_C06A63DF0E509539-001/tempDir-001/node4/deleteByCount_shard1_replica_n3], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteReplicaTest_C06A63DF0E509539-001/tempDir-001/node4/./deleteByCount_shard1_replica_n3/data/]
   [junit4]   2> 3072615 INFO  (qtp524008679-25945) [n:127.0.0.1:38261_solr c:deleteByCount s:shard1 r:core_node5 x:deleteByCount_shard1_replica_n2] o.a.s.s.IndexSchema [deleteByCount_shard1_replica_n2] Schema name=minimal
   [junit4]   2> 3072617 INFO  (qtp524008679-25945) [n:127.0.0.1:38261_solr c:deleteByCount s:shard1 r:core_node5 x:deleteByCount_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3072617 INFO  (qtp524008679-25945) [n:127.0.0.1:38261_solr c:deleteByCount s:shard1 r:core_node5 x:deleteByCount_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'deleteByCount_shard1_replica_n2' using configuration from collection deleteByCount, trusted=true
   [junit4]   2> 3072631 INFO  (qtp1756050340-25920) [n:127.0.0.1:33253_solr c:deleteByCount s:shard1 r:core_node4 x:deleteByCount_shard1_replica_n1] o.a.s.s.IndexSchema [deleteByCount_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 3072633 INFO  (qtp1756050340-25920) [n:127.0.0.1:33253_solr c:deleteByCount s:shard1 r:core_node4 x:deleteByCount_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 3072633 INFO  (qtp1756050340-25920) [n:127.0.0.1:33253_solr c:deleteByCount s:shard1 r:core_node4 x:deleteByCount_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'deleteByCount_shard1_replica_n1' using configuration from collection deleteByCount, trusted=true
   [junit4]   2> 3072666 INFO  (qtp524008679-25945) [n:127.0.0.1:38261_solr c:deleteByCount s:shard1 r:core_node5 x:deleteByCount_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38261.solr.core.deleteByCount.shard1.replica_n2' (registry 'solr.core.deleteByCount.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e998352
   [junit4]   2> 3072666 INFO  (qtp524008679-25945) [n:127.0.0.1:38261_solr c:deleteByCount s:shard1 r:core_node5 x:deleteByCount_shard1_replica_n2] o.a.s.c.SolrCore [[deleteByCount_shard1_replica_n2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteReplicaTest_C06A63DF0E509539-001/tempDir-001/node3/deleteByCount_shard1_replica_n2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteReplicaTest_C06A63DF0E509539-001/tempDir-001/node3/./deleteByCount_shard1_replica_n2/data/]
   [junit4]   2> 3072668 INFO  (qtp1756050340-25920) [n:127.0.0.1:33253_solr c:deleteByCount s:shard1 r:core_node4 x:deleteByCount_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33253.solr.core.deleteByCount.shard1.replica_n1' (registry 'solr.core.deleteByCount.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e998352
   [junit4]   2> 3072668 INFO  (qtp1756050340-25920) [n:127.0.0.1:33253_solr c:deleteByCount s:shard1 r:core_node4 x:deleteByCount_shard1_replica_n1] o.a.s.c.SolrCore [[deleteByCount_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteReplicaTest_C06A63DF0E509539-001/tempDir-001/node1/deleteByCount_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteReplicaTest_C06A63DF0E509539-001/tempDir-001/node1/./deleteByCount_shard1_replica_n1/data/]
   [junit4]   2> 3073064 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 3073064 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 3073066 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 3073066 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 3073083 INFO  (qtp524008679-25945) [n:127.0.0.1:38261_solr c:deleteByCount s:shard1 r:core_node5 x:deleteByCount_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 3073083 INFO  (qtp524008679-25945) [n:127.0.0.1:38261_solr c:deleteByCount s:shard1 r:core_node5 x:deleteByCount_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 3073097 INFO  (qtp524008679-25945) [n:127.0.0.1:38261_solr c:deleteByCount s:shard1 r:core_node5 x:deleteByCount_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 3073100 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.s.SolrIndexSearcher Opening [Searcher@14e3d626[deleteByCount_shard1_replica_n3] main]
   [junit4]   2> 3073101 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 3073101 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 3073101 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 3073101 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1630260753788305408
   [junit4]   2> 3073116 INFO  (qtp524008679-25945) [n:127.0.0.1:38261_solr c:deleteByCount s:shard1 r:core_node5 x:deleteByCount_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 3073116 INFO  (searcherExecutor-7702-thread-1-processing-n:127.0.0.1:40248_solr x:deleteByCount_shard1_replica_n3 c:deleteByCount s:shard1 r:core_node6) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.c.SolrCore [deleteByCount_shard1_replica_n3] Registered new searcher Searcher@14e3d626[deleteByCount_shard1_replica_n3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 3073116 INFO  (qtp524008679-25945) [n:127.0.0.1:38261_solr c:deleteByCount s:shard1 r:core_node5 x:deleteByCount_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@235c7a18[deleteByCount_shard1_replica_n2] main]
   [junit4]   2> 3073117 INFO  (qtp524008679-25945) [n:127.0.0.1:38261_solr c:deleteByCount s:shard1 r:core_node5 x:deleteByCount_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 3073118 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.c.ZkShardTerms Successful update of terms at /collections/deleteByCount/terms/shard1 to Terms{values={core_node6=0}, version=0}
   [junit4]   2> 3073118 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/deleteByCount/leaders/shard1
   [junit4]   2> 3073132 INFO  (qtp524008679-25945) [n:127.0.0.1:38261_solr c:deleteByCount s:shard1 r:core_node5 x:deleteByCount_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 3073132 INFO  (qtp524008679-25945) [n:127.0.0.1:38261_solr c:deleteByCount s:shard1 r:core_node5 x:deleteByCount_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 3073132 INFO  (qtp524008679-25945) [n:127.0.0.1:38261_solr c:deleteByCount s:shard1 r:core_node5 x:deleteByCount_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1630260753820811264
   [junit4]   2> 3073137 INFO  (qtp389308199-25936) [n:127.0.0.1:40248_solr c:deleteByCount s:shard1 r:core_node6 x:deleteByCount_shard1_replica_n3] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=3 found=1 timeoutin=9999ms
   [junit4]   2> 3073140 INFO  (qtp1756050340-25920) [n:127.0.0.1:33253_solr c:deleteByCount s:shard1 r:core_node4 x:deleteByCount_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 3073140 INFO  (qtp1756050340-25920) [n:127.0.0.1:33253_solr c:deleteByCount s:shard1 r:core_node4 x:deleteByCount_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 3073142 INFO  (qtp1756050340-25920) [n:127.0.0.1:33253_solr c:deleteByCount s:shard1 r:core_node4 x:deleteByCount_shard1_replica_n1] o.a.s.u.CommitTracker Hard Au

[...truncated too long message...]

.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/null501316920
     [copy] Copying 240 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/null501316920
   [delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-8.x/solr/null501316920

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] 	confs: [default]
[ivy:cachepath] 	found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] 	found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] 	found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] 	found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] 	found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] 	found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 20ms :: artifacts dl 1ms
	---------------------------------------------------------------------
	|                  |            modules            ||   artifacts   |
	|       conf       | number| search|dwnlded|evicted|| number|dwnlded|
	---------------------------------------------------------------------
	|      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
	---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 229 minutes 0 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2847)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2726)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2707)
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3086)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.FilePath.act(FilePath.java:1072)
		at hudson.FilePath.act(FilePath.java:1061)
		at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
		at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
		at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
		at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
		at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
		at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
		at hudson.model.Build$BuildExecution.post2(Build.java:186)
		at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
		at hudson.model.Run.execute(Run.java:1835)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3088)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath.act(FilePath.java:1074)
	at hudson.FilePath.act(FilePath.java:1061)
	at hudson.FilePath.validateAntFileMask(FilePath.java:2705)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
	at hudson.model.Build$BuildExecution.post2(Build.java:186)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
	at hudson.model.Run.execute(Run.java:1835)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)