You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2021/10/13 15:20:29 UTC

[JENKINS] Lucene » Lucene-Solr-Tests-8.10 - Build # 206 - Unstable!

Build: https://ci-builds.apache.org/job/Lucene/job/Lucene-Solr-Tests-8.10/206/

1 tests failed.
FAILED:  org.apache.solr.common.cloud.PerReplicaStatesIntegrationTest.testPerReplicaStateCollection

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:41099/solr]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:41099/solr]
	at __randomizedtesting.SeedInfo.seed([FC6C846E1EF33AA2:D6512F2CBE53E0AF]:0)
	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:346)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1171)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:934)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:866)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:214)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:231)
	at org.apache.solr.common.cloud.PerReplicaStatesIntegrationTest.testPerReplicaStateCollection(PerReplicaStatesIntegrationTest.java:84)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:41099/solr
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:692)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:369)
	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:297)
	... 49 more
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:571)
	... 53 more




Build Log:
[...truncated 17730 lines...]
   [junit4] Suite: org.apache.solr.common.cloud.PerReplicaStatesIntegrationTest
   [junit4]   2> 273577 INFO  (SUITE-PerReplicaStatesIntegrationTest-seed#[FC6C846E1EF33AA2]-worker) [     ] o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to test-framework derived value of '/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr/server/solr/configsets/_default/conf'
   [junit4]   2> 273577 INFO  (SUITE-PerReplicaStatesIntegrationTest-seed#[FC6C846E1EF33AA2]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 273578 INFO  (SUITE-PerReplicaStatesIntegrationTest-seed#[FC6C846E1EF33AA2]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr/build/solr-solrj/test/J2/temp/solr.common.cloud.PerReplicaStatesIntegrationTest_FC6C846E1EF33AA2-001/data-dir-22-001
   [junit4]   2> 273578 WARN  (SUITE-PerReplicaStatesIntegrationTest-seed#[FC6C846E1EF33AA2]-worker) [     ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=8 numCloses=8
   [junit4]   2> 273578 INFO  (SUITE-PerReplicaStatesIntegrationTest-seed#[FC6C846E1EF33AA2]-worker) [     ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 273580 INFO  (SUITE-PerReplicaStatesIntegrationTest-seed#[FC6C846E1EF33AA2]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 273583 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.SolrTestCaseJ4 ###Starting testRestart
   [junit4]   2> 273583 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr/build/solr-solrj/test/J2/temp/solr.common.cloud.PerReplicaStatesIntegrationTest_FC6C846E1EF33AA2-001/tempDir-001
   [junit4]   2> 273584 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 273584 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port: 0.0.0.0/0.0.0.0:0
   [junit4]   2> 273584 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 273585 WARN  (ZkTestServer Run Thread) [     ] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
   [junit4]   2> 273684 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.ZkTestServer start zk server on port: 38939
   [junit4]   2> 273684 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:38939
   [junit4]   2> 273684 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:38939
   [junit4]   2> 273684 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 38939
   [junit4]   2> 273689 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 273692 INFO  (zkConnectionManagerCallback-3994-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 273692 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 273700 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 273702 INFO  (zkConnectionManagerCallback-3996-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 273702 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 273703 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 273704 INFO  (zkConnectionManagerCallback-3998-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 273704 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 273806 INFO  (jetty-launcher-3999-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 273806 INFO  (jetty-launcher-3999-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
   [junit4]   2> 273806 INFO  (jetty-launcher-3999-thread-1) [     ] o.e.j.s.Server jetty-9.4.44.v20210927; built: 2021-09-27T23:02:44.612Z; git: 8da83308eeca865e495e53ef315a249d63ba9332; jvm 1.8.0_291-b10
   [junit4]   2> 273807 INFO  (jetty-launcher-3999-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 273807 INFO  (jetty-launcher-3999-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 273807 INFO  (jetty-launcher-3999-thread-1) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 273807 INFO  (jetty-launcher-3999-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4d38b701{/solr,null,AVAILABLE}
   [junit4]   2> 273808 INFO  (jetty-launcher-3999-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@eed4736{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:36715}
   [junit4]   2> 273808 INFO  (jetty-launcher-3999-thread-1) [     ] o.e.j.s.Server Started @273989ms
   [junit4]   2> 273808 INFO  (jetty-launcher-3999-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, zkHost=127.0.0.1:38939/solr, hostPort=36715}
   [junit4]   2> 273808 ERROR (jetty-launcher-3999-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 273808 INFO  (jetty-launcher-3999-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 273808 INFO  (jetty-launcher-3999-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.10.1
   [junit4]   2> 273808 INFO  (jetty-launcher-3999-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 273808 INFO  (jetty-launcher-3999-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr
   [junit4]   2> 273808 INFO  (jetty-launcher-3999-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2021-10-13T15:01:35.633Z
   [junit4]   2> 273809 INFO  (jetty-launcher-3999-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 273810 INFO  (zkConnectionManagerCallback-4001-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 273810 INFO  (jetty-launcher-3999-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 273810 INFO  (jetty-launcher-3999-thread-1) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 273815 INFO  (jetty-launcher-3999-thread-1) [     ] o.a.s.c.SolrXmlConfig Metrics collection is disabled.
   [junit4]   2> 277120 INFO  (jetty-launcher-3999-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 277122 WARN  (jetty-launcher-3999-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@35947583[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 277122 WARN  (jetty-launcher-3999-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@35947583[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 277127 WARN  (jetty-launcher-3999-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@336b9d0e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 277127 WARN  (jetty-launcher-3999-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@336b9d0e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 277128 INFO  (jetty-launcher-3999-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38939/solr
   [junit4]   2> 277128 INFO  (jetty-launcher-3999-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 277129 INFO  (zkConnectionManagerCallback-4012-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 277129 INFO  (jetty-launcher-3999-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 277231 INFO  (jetty-launcher-3999-thread-1) [n:127.0.0.1:36715_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 277232 INFO  (zkConnectionManagerCallback-4014-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 277232 INFO  (jetty-launcher-3999-thread-1) [n:127.0.0.1:36715_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 277300 WARN  (jetty-launcher-3999-thread-1) [n:127.0.0.1:36715_solr     ] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/zookeeper-access-control.html
   [junit4]   2> 277304 INFO  (jetty-launcher-3999-thread-1) [n:127.0.0.1:36715_solr     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:36715_solr
   [junit4]   2> 277305 INFO  (jetty-launcher-3999-thread-1) [n:127.0.0.1:36715_solr     ] o.a.s.c.Overseer Overseer (id=72571833261752325-127.0.0.1:36715_solr-n_0000000000) starting
   [junit4]   2> 277309 INFO  (OverseerStateUpdate-72571833261752325-127.0.0.1:36715_solr-n_0000000000) [n:127.0.0.1:36715_solr     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:36715_solr
   [junit4]   2> 277314 INFO  (jetty-launcher-3999-thread-1) [n:127.0.0.1:36715_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36715_solr
   [junit4]   2> 277316 INFO  (jetty-launcher-3999-thread-1) [n:127.0.0.1:36715_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 277316 WARN  (jetty-launcher-3999-thread-1) [n:127.0.0.1:36715_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 277322 INFO  (zkCallback-4013-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 277365 INFO  (jetty-launcher-3999-thread-1) [n:127.0.0.1:36715_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr/build/solr-solrj/test/J2/temp/solr.common.cloud.PerReplicaStatesIntegrationTest_FC6C846E1EF33AA2-001/tempDir-001/node1
   [junit4]   2> 277376 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=1
   [junit4]   2> 277377 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 277379 INFO  (zkConnectionManagerCallback-4029-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 277379 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 277382 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 277382 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38939/solr ready
   [junit4]   2> 277394 INFO  (OverseerThreadFactory-4021-thread-1-processing-n:127.0.0.1:36715_solr) [n:127.0.0.1:36715_solr     ] o.a.s.c.a.c.CreateCollectionCmd Create collection prs_restart_test
   [junit4]   2> 277395 DEBUG (OverseerThreadFactory-4021-thread-1-processing-n:127.0.0.1:36715_solr) [n:127.0.0.1:36715_solr     ] o.a.s.c.c.PerReplicaStatesOps touchChildren [ADD : .dummy.7846671653848882:0:D, DELETE : .dummy.7846671653848882:0:D]
   [junit4]   2> 277397 INFO  (OverseerThreadFactory-4021-thread-1-processing-n:127.0.0.1:36715_solr) [n:127.0.0.1:36715_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "core":"prs_restart_test_shard1_replica_n1",
   [junit4]   2>   "node_name":"127.0.0.1:36715_solr",
   [junit4]   2>   "base_url":"http://127.0.0.1:36715/solr",
   [junit4]   2>   "collection":"prs_restart_test",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 277410 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr    x:prs_restart_test_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=prs_restart_test_shard1_replica_n1&action=CREATE&numShards=1&collection=prs_restart_test&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 277411 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr    x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient core cache for max 2147483647 cores with initial capacity of 1024
   [junit4]   2> 277419 DEBUG (OverseerStateUpdate-72571833261752325-127.0.0.1:36715_solr-n_0000000000) [n:127.0.0.1:36715_solr     ] o.a.s.c.c.PerReplicaStatesOps flipState on /collections/prs_restart_test/state.json, core_node2 -> down, ops :[ADD : core_node2:0:D]
   [junit4]   2> 277419 DEBUG (OverseerStateUpdate-72571833261752325-127.0.0.1:36715_solr-n_0000000000) [n:127.0.0.1:36715_solr     ] o.a.s.c.c.PerReplicaStatesOps Per-replica state being persisted for : '/collections/prs_restart_test/state.json', ops: [ADD : core_node2:0:D]
   [junit4]   2> 277423 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.10.1
   [junit4]   2> 277426 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.s.IndexSchema Schema name=test
   [junit4]   2> 277428 WARN  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.D.s.TrieDateField Solr loaded a deprecated plugin/analysis class [solr.TrieDateField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 277433 WARN  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.D.s.TrieFloatField Solr loaded a deprecated plugin/analysis class [solr.TrieFloatField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 277436 WARN  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.D.s.TrieLongField Solr loaded a deprecated plugin/analysis class [solr.TrieLongField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 277438 WARN  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.D.s.TrieDoubleField Solr loaded a deprecated plugin/analysis class [solr.TrieDoubleField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 277439 WARN  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.s.IndexSchema Field lowerfilt1and2 is not multivalued and destination for multiople copyFields (2)
   [junit4]   2> 277439 WARN  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.s.IndexSchema Field text is not multivalued and destination for multiople copyFields (3)
   [junit4]   2> 277439 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 277439 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'prs_restart_test_shard1_replica_n1' using configuration from configset conf, trusted=true
   [junit4]   2> 277440 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.SolrCore [[prs_restart_test_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr/build/solr-solrj/test/J2/temp/solr.common.cloud.PerReplicaStatesIntegrationTest_FC6C846E1EF33AA2-001/tempDir-001/node1/prs_restart_test_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr/build/solr-solrj/test/J2/temp/solr.common.cloud.PerReplicaStatesIntegrationTest_FC6C846E1EF33AA2-001/tempDir-001/node1/prs_restart_test_shard1_replica_n1/data/]
   [junit4]   2> 277471 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 277471 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 277472 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 277472 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 277473 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 277473 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 277473 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
   [junit4]   2> 277473 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1713517152748699648
   [junit4]   2> 277476 INFO  (searcherExecutor-4031-thread-1-processing-n:127.0.0.1:36715_solr x:prs_restart_test_shard1_replica_n1 c:prs_restart_test s:shard1 r:core_node2) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.SolrCore [prs_restart_test_shard1_replica_n1]  Registered new searcher autowarm time: 0 ms
   [junit4]   2> 277481 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/prs_restart_test/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 277481 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/prs_restart_test/leaders/shard1
   [junit4]   2> 277485 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 277485 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 277485 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:36715/solr/prs_restart_test_shard1_replica_n1/
   [junit4]   2> 277485 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 277485 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.SyncStrategy http://127.0.0.1:36715/solr/prs_restart_test_shard1_replica_n1/ has no replicas
   [junit4]   2> 277485 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/prs_restart_test/leaders/shard1/leader after winning as /collections/prs_restart_test/leader_elect/shard1/election/72571833261752325-core_node2-n_0000000000
   [junit4]   2> 277487 DEBUG (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.c.PerReplicaStatesOps flipLeader on:/collections/prs_restart_test/state.json, [core_node2] -> core_node2, ops: [ADD : core_node2:1:A:L, DELETE : core_node2:0:D]
   [junit4]   2> 277487 DEBUG (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.c.PerReplicaStatesOps Per-replica state being persisted for : '/collections/prs_restart_test/state.json', ops: [ADD : core_node2:1:A:L, DELETE : core_node2:0:D]
   [junit4]   2> 277487 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:36715/solr/prs_restart_test_shard1_replica_n1/ shard1
   [junit4]   2> 277491 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 277491 DEBUG (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.c.PerReplicaStatesOps flipState on /collections/prs_restart_test/state.json, core_node2 -> active, ops :[ADD : core_node2:2:A:L, DELETE : core_node2:1:A:L]
   [junit4]   2> 277491 DEBUG (qtp2799939-7528) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.c.PerReplicaStatesOps Per-replica state being persisted for : '/collections/prs_restart_test/state.json', ops: [ADD : core_node2:2:A:L, DELETE : core_node2:1:A:L]
   [junit4]   2> 277495 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=prs_restart_test_shard1_replica_n1&action=CREATE&numShards=1&collection=prs_restart_test&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=84
   [junit4]   2> 277497 INFO  (qtp2799939-7525) [n:127.0.0.1:36715_solr     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 277497 INFO  (qtp2799939-7525) [n:127.0.0.1:36715_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf&name=prs_restart_test&nrtReplicas=1&action=CREATE&numShards=1&perReplicaState=true&wt=javabin&version=2} status=0 QTime=110
   [junit4]   2> 277497 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.MiniSolrCloudCluster waitForActiveCollection: prs_restart_test
   [junit4]   2> 277502 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 277502 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
   [junit4]   2> 277502 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.s.Server jetty-9.4.44.v20210927; built: 2021-09-27T23:02:44.612Z; git: 8da83308eeca865e495e53ef315a249d63ba9332; jvm 1.8.0_291-b10
   [junit4]   2> 277513 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 277514 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 277514 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 277514 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2a3738f2{/solr,null,AVAILABLE}
   [junit4]   2> 277514 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.s.AbstractConnector Started ServerConnector@5fa11f97{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:37873}
   [junit4]   2> 277514 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.s.Server Started @277696ms
   [junit4]   2> 277514 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, zkHost=127.0.0.1:38939/solr, hostPort=37873}
   [junit4]   2> 277515 ERROR (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 277515 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 277515 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.10.1
   [junit4]   2> 277515 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 277515 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr
   [junit4]   2> 277515 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2021-10-13T15:01:39.340Z
   [junit4]   2> 277516 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 277518 INFO  (zkConnectionManagerCallback-4037-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 277518 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 277518 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 277521 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.SolrXmlConfig Metrics collection is disabled.
   [junit4]   2> 277522 WARN  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]-SendThread(127.0.0.1:38939)) [     ] o.a.z.ClientCnxn An exception was thrown while closing send thread for session 0x101d3b2a6440007.
   [junit4]   2>           => EndOfStreamException: Unable to read additional data from server sessionid 0x101d3b2a6440007, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:77)
   [junit4]   2> org.apache.zookeeper.ClientCnxn$EndOfStreamException: Unable to read additional data from server sessionid 0x101d3b2a6440007, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:77) ~[zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:350) ~[zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1275) [zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 277958 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 277962 WARN  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@2e7ef7f4[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 277962 WARN  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@2e7ef7f4[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 277966 WARN  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@3a376f27[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 277966 WARN  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@3a376f27[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 277967 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38939/solr
   [junit4]   2> 277969 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 277974 INFO  (zkConnectionManagerCallback-4048-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 277974 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 278089 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 278096 INFO  (zkConnectionManagerCallback-4050-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 278096 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 278099 WARN  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/zookeeper-access-control.html
   [junit4]   2> 278111 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 278114 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:37873_solr as DOWN
   [junit4]   2> 278115 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient core cache for max 2147483647 cores with initial capacity of 1024
   [junit4]   2> 278115 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37873_solr
   [junit4]   2> 278116 INFO  (zkCallback-4013-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 278119 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 278120 WARN  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 278126 INFO  (zkCallback-4028-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 278133 INFO  (zkCallback-4049-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 278162 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr/build/solr-solrj/test/J2/temp/solr.common.cloud.PerReplicaStatesIntegrationTest_FC6C846E1EF33AA2-001/tempDir-001/node2
   [junit4]   2> 278194 INFO  (OverseerCollectionConfigSetProcessor-72571833261752325-127.0.0.1:36715_solr-n_0000000000) [n:127.0.0.1:36715_solr     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
   [junit4]   2> 278195 INFO  (OverseerThreadFactory-4021-thread-2-processing-n:127.0.0.1:36715_solr) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1   ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:37873_solr for creating new replica of shard shard1 for collection prs_restart_test
   [junit4]   2> 278197 INFO  (OverseerThreadFactory-4021-thread-2-processing-n:127.0.0.1:36715_solr) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1   ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
   [junit4]   2> 278198 INFO  (OverseerStateUpdate-72571833261752325-127.0.0.1:36715_solr-n_0000000000) [n:127.0.0.1:36715_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "core":"prs_restart_test_shard1_replica_n3",
   [junit4]   2>   "node_name":"127.0.0.1:37873_solr",
   [junit4]   2>   "base_url":"http://127.0.0.1:37873/solr",
   [junit4]   2>   "collection":"prs_restart_test",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "operation":"addreplica"} 
   [junit4]   2> 278199 DEBUG (OverseerStateUpdate-72571833261752325-127.0.0.1:36715_solr-n_0000000000) [n:127.0.0.1:36715_solr     ] o.a.s.c.c.PerReplicaStatesOps Per-replica state being persisted for : '/collections/prs_restart_test/state.json', ops: [ADD : core_node4:0:D]
   [junit4]   2> 278199 INFO  (zkCallback-4013-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 278200 INFO  (zkCallback-4013-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 278214 INFO  (zkCallback-4013-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 278214 INFO  (zkCallback-4013-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 278339 INFO  (qtp1626851207-7571) [n:127.0.0.1:37873_solr    x:prs_restart_test_shard1_replica_n3 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&name=prs_restart_test_shard1_replica_n3&action=CREATE&collection=prs_restart_test&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 278365 DEBUG (OverseerStateUpdate-72571833261752325-127.0.0.1:36715_solr-n_0000000000) [n:127.0.0.1:36715_solr     ] o.a.s.c.c.PerReplicaStatesOps flipState on /collections/prs_restart_test/state.json, core_node4 -> down, ops :[ADD : core_node4:1:D, DELETE : core_node4:0:D]
   [junit4]   2> 278365 DEBUG (OverseerStateUpdate-72571833261752325-127.0.0.1:36715_solr-n_0000000000) [n:127.0.0.1:36715_solr     ] o.a.s.c.c.PerReplicaStatesOps Per-replica state being persisted for : '/collections/prs_restart_test/state.json', ops: [ADD : core_node4:1:D, DELETE : core_node4:0:D]
   [junit4]   2> 278365 INFO  (zkCallback-4013-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 278367 INFO  (zkCallback-4013-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 278367 INFO  (zkCallback-4013-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 278367 INFO  (zkCallback-4013-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 278371 INFO  (qtp1626851207-7571) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.10.1
   [junit4]   2> 278374 INFO  (qtp1626851207-7571) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.s.IndexSchema Schema name=test
   [junit4]   2> 278377 WARN  (qtp1626851207-7571) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.s.IndexSchema Field lowerfilt1and2 is not multivalued and destination for multiople copyFields (2)
   [junit4]   2> 278386 WARN  (qtp1626851207-7571) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.s.IndexSchema Field text is not multivalued and destination for multiople copyFields (3)
   [junit4]   2> 278386 INFO  (qtp1626851207-7571) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 278386 INFO  (qtp1626851207-7571) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.CoreContainer Creating SolrCore 'prs_restart_test_shard1_replica_n3' using configuration from configset conf, trusted=true
   [junit4]   2> 278386 INFO  (qtp1626851207-7571) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.SolrCore [[prs_restart_test_shard1_replica_n3] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr/build/solr-solrj/test/J2/temp/solr.common.cloud.PerReplicaStatesIntegrationTest_FC6C846E1EF33AA2-001/tempDir-001/node2/prs_restart_test_shard1_replica_n3], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr/build/solr-solrj/test/J2/temp/solr.common.cloud.PerReplicaStatesIntegrationTest_FC6C846E1EF33AA2-001/tempDir-001/node2/prs_restart_test_shard1_replica_n3/data/]
   [junit4]   2> 278432 INFO  (qtp1626851207-7571) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 278432 INFO  (qtp1626851207-7571) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 278432 INFO  (qtp1626851207-7571) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 278432 INFO  (qtp1626851207-7571) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 278638 INFO  (qtp1626851207-7571) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 278638 INFO  (qtp1626851207-7571) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 278638 INFO  (qtp1626851207-7571) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
   [junit4]   2> 278638 INFO  (qtp1626851207-7571) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1713517153970290688
   [junit4]   2> 278641 INFO  (searcherExecutor-4058-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.SolrCore [prs_restart_test_shard1_replica_n3]  Registered new searcher autowarm time: 0 ms
   [junit4]   2> 278649 INFO  (qtp1626851207-7571) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/prs_restart_test/terms/shard1 to Terms{values={core_node2=0, core_node4=0}, version=1}
   [junit4]   2> 278649 INFO  (qtp1626851207-7571) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/prs_restart_test/leaders/shard1
   [junit4]   2> 278653 INFO  (qtp1626851207-7571) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.ZkController Core needs to recover:prs_restart_test_shard1_replica_n3
   [junit4]   2> 278657 INFO  (updateExecutor-4044-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 278657 INFO  (qtp1626851207-7571) [n:127.0.0.1:37873_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&name=prs_restart_test_shard1_replica_n3&action=CREATE&collection=prs_restart_test&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=318
   [junit4]   2> 278658 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 278658 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy startupVersions is empty
   [junit4]   2> 278660 INFO  (qtp2799939-7527) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.S.Request [prs_restart_test_shard1_replica_n1]  webapp=/solr path=/admin/ping params={wt=javabin&version=2} hits=0 status=0 QTime=0
   [junit4]   2> 278660 INFO  (qtp2799939-7527) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.S.Request [prs_restart_test_shard1_replica_n1]  webapp=/solr path=/admin/ping params={wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 278661 INFO  (qtp2799939-7529) [n:127.0.0.1:36715_solr c:prs_restart_test    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=ADDREPLICA&collection=prs_restart_test&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=479
   [junit4]   2> 278661 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[prs_restart_test_shard1_replica_n3]
   [junit4]   2> 278661 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 278661 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Publishing state of core [prs_restart_test_shard1_replica_n3] as recovering, leader is [http://127.0.0.1:36715/solr/prs_restart_test_shard1_replica_n1/] and I am [http://127.0.0.1:37873/solr/prs_restart_test_shard1_replica_n3/]
   [junit4]   2> 278661 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.MiniSolrCloudCluster waitForActiveCollection: prs_restart_test
   [junit4]   2> 278661 DEBUG (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.c.PerReplicaStatesOps flipState on /collections/prs_restart_test/state.json, core_node4 -> recovering, ops :[ADD : core_node4:2:R, DELETE : core_node4:1:D]
   [junit4]   2> 278661 DEBUG (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.c.PerReplicaStatesOps Per-replica state being persisted for : '/collections/prs_restart_test/state.json', ops: [ADD : core_node4:2:R, DELETE : core_node4:1:D]
   [junit4]   2> 278662 INFO  (zkCallback-4013-thread-4) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 278662 INFO  (zkCallback-4049-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 278662 INFO  (zkCallback-4028-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 278665 INFO  (zkCallback-4049-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 278666 INFO  (zkCallback-4049-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 278670 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:36715/solr]; [WaitForState: action=PREPRECOVERY&core=prs_restart_test_shard1_replica_n1&nodeName=127.0.0.1:37873_solr&coreNodeName=core_node4&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 278670 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr    x:prs_restart_test_shard1_replica_n1 ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node4, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
   [junit4]   2> 278670 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr     ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=prs_restart_test, shard=shard1, thisCore=prs_restart_test_shard1_replica_n1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:37873_solr, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{
   [junit4]   2>   "core":"prs_restart_test_shard1_replica_n3",
   [junit4]   2>   "node_name":"127.0.0.1:37873_solr",
   [junit4]   2>   "base_url":"http://127.0.0.1:37873/solr",
   [junit4]   2>   "state":"recovering",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "force_set_state":"false"}
   [junit4]   2> 278670 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr     ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=prs_restart_test, shard=shard1, thisCore=prs_restart_test_shard1_replica_n1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:37873_solr, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{
   [junit4]   2>   "core":"prs_restart_test_shard1_replica_n3",
   [junit4]   2>   "node_name":"127.0.0.1:37873_solr",
   [junit4]   2>   "base_url":"http://127.0.0.1:37873/solr",
   [junit4]   2>   "state":"recovering",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "force_set_state":"false"}
   [junit4]   2> 278670 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr     ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=prs_restart_test, shard=shard1, thisCore=prs_restart_test_shard1_replica_n1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:37873_solr, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{
   [junit4]   2>   "core":"prs_restart_test_shard1_replica_n3",
   [junit4]   2>   "node_name":"127.0.0.1:37873_solr",
   [junit4]   2>   "base_url":"http://127.0.0.1:37873/solr",
   [junit4]   2>   "state":"recovering",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "force_set_state":"false"}
   [junit4]   2> 278670 INFO  (qtp2799939-7528) [n:127.0.0.1:36715_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:37873_solr&onlyIfLeaderActive=true&core=prs_restart_test_shard1_replica_n1&coreNodeName=core_node4&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 279171 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:36715/solr/prs_restart_test_shard1_replica_n1/] - recoveringAfterStartup=[true]
   [junit4]   2> 279171 WARN  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.u.PeerSyncWithLeader no frame of reference to tell if we've missed updates
   [junit4]   2> 279171 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy PeerSync Recovery was not successful - trying replication.
   [junit4]   2> 279171 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Starting Replication Recovery.
   [junit4]   2> 279171 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Attempting to replicate from [http://127.0.0.1:36715/solr/prs_restart_test_shard1_replica_n1/].
   [junit4]   2> 279184 INFO  (qtp1626851207-7569) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.u.p.DistributedUpdateProcessor Ignoring commit while not ACTIVE - state: BUFFERING replay: false
   [junit4]   2> 279184 INFO  (qtp1626851207-7569) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.S.Request [prs_restart_test_shard1_replica_n3]  webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=false&commit=true&softCommit=false&distrib.from=http://127.0.0.1:36715/solr/prs_restart_test_shard1_replica_n1/&commit_end_point=replicas&wt=javabin&version=2&expungeDeletes=false} status=0 QTime=1
   [junit4]   2> 279184 INFO  (qtp2799939-7525) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.S.Request [prs_restart_test_shard1_replica_n1]  webapp=/solr path=/update params={waitSearcher=true&openSearcher=false&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=12
   [junit4]   2> 279186 INFO  (qtp2799939-7524) [n:127.0.0.1:36715_solr c:prs_restart_test s:shard1 r:core_node2 x:prs_restart_test_shard1_replica_n1 ] o.a.s.c.S.Request [prs_restart_test_shard1_replica_n1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 279186 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.h.IndexFetcher Leader's generation: 1
   [junit4]   2> 279186 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.h.IndexFetcher Leader's version: 0
   [junit4]   2> 279186 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.h.IndexFetcher Follower's generation: 1
   [junit4]   2> 279186 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.h.IndexFetcher Follower's version: 0
   [junit4]   2> 279186 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.h.IndexFetcher New index in Leader. Deleting mine...
   [junit4]   2> 279188 INFO  (searcherExecutor-4058-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.SolrCore [prs_restart_test_shard1_replica_n3]  Registered new searcher autowarm time: 0 ms
   [junit4]   2> 279188 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy No replay needed.
   [junit4]   2> 279188 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Replication Recovery was successful.
   [junit4]   2> 279188 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
   [junit4]   2> 279188 DEBUG (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.c.PerReplicaStatesOps flipState on /collections/prs_restart_test/state.json, core_node4 -> active, ops :[ADD : core_node4:3:A, DELETE : core_node4:2:R]
   [junit4]   2> 279188 DEBUG (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.c.PerReplicaStatesOps Per-replica state being persisted for : '/collections/prs_restart_test/state.json', ops: [ADD : core_node4:3:A, DELETE : core_node4:2:R]
   [junit4]   2> 279189 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Updating version bucket highest from index after successful recovery.
   [junit4]   2> 279189 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1713517154548056064
   [junit4]   2> 279189 INFO  (zkCallback-4028-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 279189 INFO  (zkCallback-4049-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 279189 INFO  (zkCallback-4013-thread-4) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 279190 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Finished recovery process, successful=[true]
   [junit4]   2> 279190 INFO  (recoveryExecutor-4046-thread-1-processing-n:127.0.0.1:37873_solr x:prs_restart_test_shard1_replica_n3 c:prs_restart_test s:shard1 r:core_node4) [n:127.0.0.1:37873_solr c:prs_restart_test s:shard1 r:core_node4 x:prs_restart_test_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Finished recovery process. recoveringAfterStartup=true msTimeTaken=532.0
   [junit4]   2> 279190 INFO  (zkCallback-4049-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 279192 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.PerReplicaStatesIntegrationTest restarting the node : 127.0.0.1:37873_solr, state.json v: 4 downreplica :core_node4
   [junit4]   2> 279192 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1337820091
   [junit4]   2> 279192 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:37873_solr
   [junit4]   2> 279193 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.ZkController Publish this node as DOWN...
   [junit4]   2> 279193 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.ZkController Publish node=127.0.0.1:37873_solr as DOWN
   [junit4]   2> 279194 DEBUG (OverseerStateUpdate-72571833261752325-127.0.0.1:36715_solr-n_0000000000) [n:127.0.0.1:36715_solr     ] o.a.s.c.c.PerReplicaStatesOps for coll: {/collections/prs_restart_test/state.json/[12]: [core_node2:2:A:L, core_node4:3:A]} down replicas [core_node4], ops [ADD : core_node4:4:D, DELETE : core_node4:3:A]
   [junit4]   2> 279195 DEBUG (OverseerStateUpdate-72571833261752325-127.0.0.1:36715_solr-n_0000000000) [n:127.0.0.1:36715_solr     ] o.a.s.c.c.PerReplicaStatesOps Per-replica state being persisted for : '/collections/prs_restart_test/state.json', ops: [ADD : core_node4:4:D, DELETE : core_node4:3:A]
   [junit4]   2> 279196 INFO  (zkCallback-4013-thread-4) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 279196 INFO  (zkCallback-4049-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 279205 INFO  (zkCallback-4049-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 279206 INFO  (zkCallback-4049-thread-4) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections/prs_restart_test/state.json] for collection [prs_restart_test] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 279209 INFO  (coreCloseExecutor-4067-thread-1) [n:127.0.0.1:37873_solr     ] o.a.s.c.SolrCore [prs_restart_test_shard1_replica_n3]  CLOSING SolrCore org.apache.solr.core.SolrCore@55382cfb
   [junit4]   2> 279209 INFO  (coreCloseExecutor-4067-thread-1) [n:127.0.0.1:37873_solr     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.prs_restart_test.shard1.replica_n3 tag=SolrCore@55382cfb
   [junit4]   2> 279209 INFO  (coreCloseExecutor-4067-thread-1) [n:127.0.0.1:37873_solr     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.prs_restart_test.shard1.leader tag=SolrCore@55382cfb
   [junit4]   2> 279210 INFO  (coreCloseExecutor-4067-thread-1) [n:127.0.0.1:37873_solr     ] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter.close()  ... SKIPPED (unnecessary).
   [junit4]   2> 279211 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node tag=null
   [junit4]   2> 279212 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm tag=null
   [junit4]   2> 279212 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty tag=null
   [junit4]   2> 279212 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster tag=null
   [junit4]   2> 279214 INFO  (zkCallback-4028-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 279214 INFO  (zkCallback-4013-thread-4) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 279429 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@5fa11f97{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:0}
   [junit4]   2> 279429 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2a3738f2{/solr,null,STOPPED}
   [junit4]   2> 279429 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 279431 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.PerReplicaStatesIntegrationTest after down node, state.json v: 4
   [junit4]   2> 279431 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=37873)
   [junit4]   2> 279431 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 37873 try number 2 ...
   [junit4]   2> 279431 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.s.Server jetty-9.4.44.v20210927; built: 2021-09-27T23:02:44.612Z; git: 8da83308eeca865e495e53ef315a249d63ba9332; jvm 1.8.0_291-b10
   [junit4]   2> 279435 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 279435 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 279435 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 279435 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2459e1fb{/solr,null,AVAILABLE}
   [junit4]   2> 279435 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.s.AbstractConnector Started ServerConnector@25f7b503{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:37873}
   [junit4]   2> 279435 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.s.Server Started @279617ms
   [junit4]   2> 279436 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, zkHost=127.0.0.1:38939/solr, hostPort=37873}
   [junit4]   2> 279436 ERROR (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 279436 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 279436 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.10.1
   [junit4]   2> 279436 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 279436 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr
   [junit4]   2> 279436 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2021-10-13T15:01:41.261Z
   [junit4]   2> 279439 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 279439 INFO  (zkConnectionManagerCallback-4071-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 279439 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 279440 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 279443 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.SolrXmlConfig Metrics collection is disabled.
   [junit4]   2> 280398 INFO  (OverseerCollectionConfigSetProcessor-72571833261752325-127.0.0.1:36715_solr-n_0000000000) [n:127.0.0.1:36715_solr     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
   [junit4]   2> 281220 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 281221 WARN  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@1879ccfb[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 281221 WARN  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@1879ccfb[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 281225 WARN  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@f2804b2[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 281225 WARN  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@f2804b2[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 281226 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38939/solr
   [junit4]   2> 281237 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 281248 INFO  (zkConnectionManagerCallback-4082-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 281248 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 281249 WARN  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]-SendThread(127.0.0.1:38939)) [     ] o.a.z.ClientCnxn An exception was thrown while closing send thread for session 0x101d3b2a644000b.
   [junit4]   2>           => EndOfStreamException: Unable to read additional data from server sessionid 0x101d3b2a644000b, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:77)
   [junit4]   2> org.apache.zookeeper.ClientCnxn$EndOfStreamException: Unable to read additional data from server sessionid 0x101d3b2a644000b, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:77) ~[zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:350) ~[zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1275) [zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 281361 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 281370 INFO  (zkConnectionManagerCallback-4084-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 281370 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 281539 WARN  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/zookeeper-access-control.html
   [junit4]   2> 281540 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 281543 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:37873_solr as DOWN
   [junit4]   2> 281544 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient core cache for max 2147483647 cores with initial capacity of 1024
   [junit4]   2> 281544 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37873_solr
   [junit4]   2> 281545 DEBUG (OverseerStateUpdate-72571833261752325-127.0.0.1:36715_solr-n_0000000000) [n:127.0.0.1:36715_solr     ] o.a.s.c.c.PerReplicaStatesOps for coll: {/collections/prs_restart_test/state.json/[14]: [core_node2:2:A:L, core_node4:4:D]} down replicas [core_node4], ops []
   [junit4]   2> 281546 INFO  (zkCallback-4028-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 281546 INFO  (zkCallback-4013-thread-4) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 281548 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 281549 WARN  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 281562 INFO  (zkCallback-4083-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 281581 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr/build/solr-solrj/test/J2/temp/solr.common.cloud.PerReplicaStatesIntegrationTest_FC6C846E1EF33AA2-001/tempDir-001/node2
   [junit4]   2> 281581 INFO  (TEST-PerReplicaStatesIntegrationTest.testRestart-seed#[FC6C846E1EF33AA2]) [n:127.0.0.1:37873_solr     ] o.a.s.c.CorePropertiesLocator Cores are: [prs_restart_test_shard1_replica_n3]
   [junit4]   2> 281599 INFO  (coreLoadExecutor-4090-thread-1-processing-n:127.0.0.1:37873_solr) [n:127.0.0.1:37873_solr     ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.10.1
   [junit4] 

[...truncated too long message...]

junit4]   2> 468411 WARN  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	9	/solr/collections/perReplicaState_test/terms/shard1
   [junit4]   2> 	5	/solr/clusterprops.json
   [junit4]   2> 	5	/solr/collections/perReplicaState_testv2/terms/shard1
   [junit4]   2> 	5	/solr/aliases.json
   [junit4]   2> 	5	/solr/collections/perReplicaState_testv2/terms/shard2
   [junit4]   2> 	5	/solr/collections/perReplicaState_test/terms/shard2
   [junit4]   2> 	4	/solr/packages.json
   [junit4]   2> 	4	/solr/security.json
   [junit4]   2> 	4	/solr/configs/conf
   [junit4]   2> 	4	/solr/collections/perReplicaState_test/collectionprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	38	/solr/collections/perReplicaState_test/state.json
   [junit4]   2> 	16	/solr/collections/perReplicaState_testv2/state.json
   [junit4]   2> 	5	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/overseer_elect/election/72571833894305799-127.0.0.1:41099_solr-n_0000000000
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	59	/solr/collections/perReplicaState_test/state.json
   [junit4]   2> 	33	/solr/collections/perReplicaState_testv2/state.json
   [junit4]   2> 	27	/solr/overseer/queue
   [junit4]   2> 	21	/solr/live_nodes
   [junit4]   2> 	14	/solr/collections
   [junit4]   2> 	10	/solr/overseer/collection-queue-work
   [junit4]   2> 
   [junit4]   2> 468414 INFO  (TEST-PerReplicaStatesIntegrationTest.testPerReplicaStateCollection-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:44967
   [junit4]   2> 468414 INFO  (TEST-PerReplicaStatesIntegrationTest.testPerReplicaStateCollection-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:44967
   [junit4]   2> 468414 INFO  (TEST-PerReplicaStatesIntegrationTest.testPerReplicaStateCollection-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 44967
   [junit4]   2> 468415 INFO  (TEST-PerReplicaStatesIntegrationTest.testPerReplicaStateCollection-seed#[FC6C846E1EF33AA2]) [     ] o.a.s.SolrTestCaseJ4 ###Ending testPerReplicaStateCollection
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=PerReplicaStatesIntegrationTest -Dtests.method=testPerReplicaStateCollection -Dtests.seed=FC6C846E1EF33AA2 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es-PE -Dtests.timezone=Asia/Yakutsk -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR    185s J2 | PerReplicaStatesIntegrationTest.testPerReplicaStateCollection <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:41099/solr]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([FC6C846E1EF33AA2:D6512F2CBE53E0AF]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:346)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1171)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:934)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:866)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:231)
   [junit4]    > 	at org.apache.solr.common.cloud.PerReplicaStatesIntegrationTest.testPerReplicaStateCollection(PerReplicaStatesIntegrationTest.java:84)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:41099/solr
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:692)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:369)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:297)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:171)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:141)
   [junit4]    > 	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
   [junit4]    > 	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
   [junit4]    > 	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
   [junit4]    > 	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
   [junit4]    > 	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
   [junit4]    > 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
   [junit4]    > 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
   [junit4]    > 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]    > 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:571)
   [junit4]    > 	... 53 more
   [junit4]   2> 468418 INFO  (SUITE-PerReplicaStatesIntegrationTest-seed#[FC6C846E1EF33AA2]-worker) [     ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
   [junit4]   2> 468419 INFO  (SUITE-PerReplicaStatesIntegrationTest-seed#[FC6C846E1EF33AA2]-worker) [     ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:44967
   [junit4]   2> 468419 INFO  (SUITE-PerReplicaStatesIntegrationTest-seed#[FC6C846E1EF33AA2]-worker) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:44967
   [junit4]   2> 468419 INFO  (SUITE-PerReplicaStatesIntegrationTest-seed#[FC6C846E1EF33AA2]-worker) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 44967
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr/build/solr-solrj/test/J2/temp/solr.common.cloud.PerReplicaStatesIntegrationTest_FC6C846E1EF33AA2-001
   [junit4]   2> Oct 13, 2021 3:04:50 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene87): {}, docValues:{}, maxPointsInLeafNode=526, maxMBSortInHeap=5.668513484598061, sim=Asserting(RandomSimilarity(queryNorm=true): {}), locale=es-PE, timezone=Asia/Yakutsk
   [junit4]   2> NOTE: Linux 4.15.0-147-generic amd64/Oracle Corporation 1.8.0_291 (64-bit)/cpus=4,threads=1,free=283172088,total=511180800
   [junit4]   2> NOTE: All tests run in this JVM: [ConcurrentUpdateSolrClientMultiCollectionTest, RawValueEvaluatorTest, QueryFacetMapTest, UsingSolrJRefGuideExamplesTest, CoalesceEvaluatorTest, TangentEvaluatorTest, MoveReplicaSuggesterTest, HttpSolrClientSSLAuthConPoolTest, TestCloudCollectionsListeners, OperationsTest, NormalizeEvaluatorTest, TestSolrJsonWriter, TestLBHttpSolrClient, DirectJsonQueryRequestFacetingEmbeddedTest, SolrQueryTest, SolrExampleBinaryHttp2Test, CommonParamsTest, LargeVolumeJettyTest, TestToleratedUpdateError, CloudHttp2SolrClientTest, GreaterThanEqualToEvaluatorTest, TestJsonRecordReader, JsonQueryRequestFacetingIntegrationTest, ModuloEvaluatorTest, SelectWithEvaluatorsTest, StreamExpressionTest, StreamExpressionToExpessionTest, StreamExpressionToExplanationTest, AbsoluteValueEvaluatorTest, AddEvaluatorTest, AndEvaluatorTest, AppendEvaluatorTest, ArcSineEvaluatorTest, CorrelationEvaluatorTest, CubedRootEvaluatorTest, EmpiricalDistributionEvaluatorTest, ExclusiveOrEvaluatorTest, HyperbolicSineEvaluatorTest, NotEvaluatorTest, ReverseEvaluatorTest, SineEvaluatorTest, UniformDistributionEvaluatorTest, SolrPingTest, TestCollectionAdminRequest, TestUpdateRequestCodec, RangeFacetMapTest, TermsFacetMapTest, AnlysisResponseBaseTest, DocumentAnalysisResponseTest, FacetFieldTest, FieldAnalysisResponseTest, NoOpResponseParserTest, TestDelegationTokenResponse, TestSuggesterResponse, ReplicaListTransformerTest, PerReplicaStatesIntegrationTest]
   [junit4] Completed [215/215 (1!)] on J2 in 195.21s, 2 tests, 1 error <<< FAILURES!

[...truncated 53088 lines...]

[JENKINS] Lucene » Lucene-Solr-Tests-8.10 - Build # 207 - Still Unstable!

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Lucene/job/Lucene-Solr-Tests-8.10/207/

1 tests failed.
FAILED:  org.apache.solr.s3.S3IncrementalBackupTest.testBackupIncremental

Error Message:
Error from server at https://127.0.0.1:44747/solr: Could not backup all shards

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:44747/solr: Could not backup all shards
	at __randomizedtesting.SeedInfo.seed([C1159D09C0742EC4:C6D032C2FC9C10B9]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:681)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:369)
	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:297)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1171)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:934)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:866)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:214)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:231)
	at org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest$IncrementalBackupVerifier.backupThenWait(AbstractIncrementalBackupTest.java:477)
	at org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest$IncrementalBackupVerifier.incrementalBackupThenVerify(AbstractIncrementalBackupTest.java:487)
	at org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest.backupRestoreThenCheck(AbstractIncrementalBackupTest.java:373)
	at org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest.testBackupIncremental(AbstractIncrementalBackupTest.java:244)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.adobe.testing.s3mock.junit4.S3MockRule$1.evaluate(S3MockRule.java:66)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 26089 lines...]
   [junit4] Suite: org.apache.solr.s3.S3IncrementalBackupTest
   [junit4]   2> 21242 INFO  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker) [    ] o.e.j.s.Server jetty-9.4.44.v20210927; built: 2021-09-27T23:02:44.612Z; git: 8da83308eeca865e495e53ef315a249d63ba9332; jvm 1.8.0_291-b10
   [junit4]   2> 21280 INFO  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker) [    ] o.e.j.s.h.C.application Initializing Spring embedded WebApplicationContext
   [junit4]   2> 21327 INFO  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 21327 INFO  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 21327 INFO  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 21343 INFO  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker) [    ] o.e.j.s.h.ContextHandler Started o.s.b.w.e.j.JettyEmbeddedWebAppContext@38a93c5e{application,/,[file:///home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr/build/contrib/solr-s3-repository/test/J2/temp/jetty-docbase.0.6258932750794238146/],AVAILABLE}
   [junit4]   2> 21344 INFO  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker) [    ] o.e.j.s.Server Started @21393ms
   [junit4]   2> 21566 INFO  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker) [    ] o.e.j.s.h.C.application Initializing Spring DispatcherServlet 'dispatcherServlet'
   [junit4]   2> 22174 INFO  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker) [    ] o.e.j.u.s.SslContextFactory x509=X509@17d58b49(selfsigned,h=[unknown],a=[],w=[]) for Server@66ee5058[provider=null,keyStore=jar:file:/home/jenkins/.ivy2/cache/com.adobe.testing/s3mock/jars/s3mock-2.1.34.jar!/s3mock.jks,trustStore=null]
   [junit4]   2> 22281 INFO  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker) [    ] o.e.j.s.AbstractConnector Started SslValidatingServerConnector@7ed39dc4{SSL, (ssl, http/1.1)}{0.0.0.0:46797}
   [junit4]   2> 22283 INFO  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker) [    ] o.e.j.s.AbstractConnector Started ServerConnector@26f3bb57{HTTP/1.1, (http/1.1)}{0.0.0.0:45119}
   [junit4]   2> 22323 WARN  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker) [    ] o.a.s.c.ZkTestServer TEST_HOME() does not exist - solrj test?
   [junit4]   2> 22421 WARN  (ZkTestServer Run Thread) [    ] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
   [junit4]   2> 22830 WARN  (NIOWorkerThread-2) [    ] o.a.z.s.NIOServerCnxn Close of session 0x0
   [junit4]   2> java.io.IOException: ZooKeeperServer not running
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.readLength(NIOServerCnxn.java:544) ~[zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:332) [zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory$IOWorkRequest.doWork(NIOServerCnxnFactory.java:522) [zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 	at org.apache.zookeeper.server.WorkerService$ScheduledWorkRequest.run(WorkerService.java:154) [zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_291]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_291]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_291]
   [junit4]   2> 22852 WARN  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker-SendThread(127.0.0.1:43063)) [    ] o.a.z.ClientCnxn Session 0x0 for sever localhost.localdomain/127.0.0.1:43063, Closing socket connection. Attempting reconnect except it is a SessionExpiredException.
   [junit4]   2> org.apache.zookeeper.ClientCnxn$EndOfStreamException: Unable to read additional data from server sessionid 0x0, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:77) ~[zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:350) ~[zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1275) [zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 24499 WARN  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker-SendThread(127.0.0.1:43063)) [    ] o.a.z.ClientCnxn An exception was thrown while closing send thread for session 0x101d4adee7c0002.
   [junit4]   2> org.apache.zookeeper.ClientCnxn$EndOfStreamException: Unable to read additional data from server sessionid 0x101d4adee7c0002, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:77) ~[zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:350) ~[zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1275) [zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 24769 INFO  (jetty-launcher-9-thread-1) [    ] o.e.j.s.Server jetty-9.4.44.v20210927; built: 2021-09-27T23:02:44.612Z; git: 8da83308eeca865e495e53ef315a249d63ba9332; jvm 1.8.0_291-b10
   [junit4]   2> 24770 INFO  (jetty-launcher-9-thread-2) [    ] o.e.j.s.Server jetty-9.4.44.v20210927; built: 2021-09-27T23:02:44.612Z; git: 8da83308eeca865e495e53ef315a249d63ba9332; jvm 1.8.0_291-b10
   [junit4]   2> 24771 INFO  (jetty-launcher-9-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 24771 INFO  (jetty-launcher-9-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 24772 INFO  (jetty-launcher-9-thread-1) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 24776 INFO  (jetty-launcher-9-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@25f5ee91{/solr,null,AVAILABLE}
   [junit4]   2> 24780 INFO  (jetty-launcher-9-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@16250f41{SSL, (ssl, http/1.1)}{127.0.0.1:44747}
   [junit4]   2> 24780 INFO  (jetty-launcher-9-thread-1) [    ] o.e.j.s.Server Started @24829ms
   [junit4]   2> 24853 INFO  (jetty-launcher-9-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 24853 INFO  (jetty-launcher-9-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 24853 INFO  (jetty-launcher-9-thread-2) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 24858 ERROR (jetty-launcher-9-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 24859 INFO  (jetty-launcher-9-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@14848477{/solr,null,AVAILABLE}
   [junit4]   2> 24867 INFO  (jetty-launcher-9-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4de0da61{SSL, (ssl, http/1.1)}{127.0.0.1:41533}
   [junit4]   2> 24867 INFO  (jetty-launcher-9-thread-2) [    ] o.e.j.s.Server Started @24916ms
   [junit4]   2> 24868 ERROR (jetty-launcher-9-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 25148 WARN  (jetty-launcher-9-thread-2-SendThread(127.0.0.1:43063)) [    ] o.a.z.ClientCnxn An exception was thrown while closing send thread for session 0x101d4adee7c0004.
   [junit4]   2> org.apache.zookeeper.ClientCnxn$EndOfStreamException: Unable to read additional data from server sessionid 0x101d4adee7c0004, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:77) ~[zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:350) ~[zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1275) [zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 25149 WARN  (jetty-launcher-9-thread-1-SendThread(127.0.0.1:43063)) [    ] o.a.z.ClientCnxn An exception was thrown while closing send thread for session 0x101d4adee7c0003.
   [junit4]   2> org.apache.zookeeper.ClientCnxn$EndOfStreamException: Unable to read additional data from server sessionid 0x101d4adee7c0003, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:77) ~[zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:350) ~[zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1275) [zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 26713 WARN  (jetty-launcher-9-thread-2) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 27060 WARN  (jetty-launcher-9-thread-2) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 27911 WARN  (jetty-launcher-9-thread-1) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 27938 WARN  (jetty-launcher-9-thread-1) [    ] o.a.s.c.s.i.Http2SolrClient Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2
   [junit4]   2> 27969 WARN  (jetty-launcher-9-thread-2) [n:127.0.0.1:41533_solr    ] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/zookeeper-access-control.html
   [junit4]   2> 28083 WARN  (jetty-launcher-9-thread-1) [n:127.0.0.1:44747_solr    ] o.a.s.c.ZkController Contents of zookeeper /security.json are world-readable; consider setting up ACLs as described in https://solr.apache.org/guide/zookeeper-access-control.html
   [junit4]   2> 28343 WARN  (jetty-launcher-9-thread-1) [n:127.0.0.1:44747_solr    ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 28417 WARN  (jetty-launcher-9-thread-2) [n:127.0.0.1:41533_solr    ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 38464 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0_Lucene84_0.pos did not match. expected checksum is 3401774990 and actual is checksum 879307049. expected length is 98 and actual length is 98
   [junit4]   2> 38465 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0_Lucene84_0.pos did not match. expected checksum is 2283759653 and actual is checksum 3331744652. expected length is 96 and actual length is 96
   [junit4]   2> 38475 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0_Lucene84_0.tim did not match. expected checksum is 689936175 and actual is checksum 921818186. expected length is 877 and actual length is 877
   [junit4]   2> 38479 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0_Lucene84_0.tim did not match. expected checksum is 1507830090 and actual is checksum 4103383709. expected length is 801 and actual length is 801
   [junit4]   2> 38489 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0_TestBloomFilteredLucenePostings_0.tmd did not match. expected checksum is 1380783089 and actual is checksum 2939495158. expected length is 301 and actual length is 301
   [junit4]   2> 38491 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0_TestBloomFilteredLucenePostings_0.tmd did not match. expected checksum is 3535791441 and actual is checksum 2361201903. expected length is 301 and actual length is 301
   [junit4]   2> 38494 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0_Lucene84_0.tip did not match. expected checksum is 4150078658 and actual is checksum 2055121895. expected length is 72 and actual length is 72
   [junit4]   2> 38495 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0.nvd did not match. expected checksum is 2030179826 and actual is checksum 2529197772. expected length is 59 and actual length is 59
   [junit4]   2> 38507 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0.nvd did not match. expected checksum is 2615021195 and actual is checksum 2113390031. expected length is 59 and actual length is 59
   [junit4]   2> 38508 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0_Lucene84_0.tip did not match. expected checksum is 2663033498 and actual is checksum 23079172. expected length is 72 and actual length is 72
   [junit4]   2> 38517 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0_Asserting_0.dvd did not match. expected checksum is 3844465710 and actual is checksum 2776131473. expected length is 98 and actual length is 98
   [junit4]   2> 38519 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0_Asserting_0.dvd did not match. expected checksum is 3405234971 and actual is checksum 2380543456. expected length is 96 and actual length is 96
   [junit4]   2> 38523 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0_TestBloomFilteredLucenePostings_0.doc did not match. expected checksum is 3140105421 and actual is checksum 966978855. expected length is 101 and actual length is 101
   [junit4]   2> 38524 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0_TestBloomFilteredLucenePostings_0.doc did not match. expected checksum is 1844616675 and actual is checksum 1205511553. expected length is 101 and actual length is 101
   [junit4]   2> 38534 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0_MockRandom_0.doc did not match. expected checksum is 3969653782 and actual is checksum 1846855164. expected length is 101 and actual length is 101
   [junit4]   2> 38534 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0_MockRandom_0.doc did not match. expected checksum is 1854339789 and actual is checksum 3485433379. expected length is 99 and actual length is 99
   [junit4]   2> 38539 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0_Asserting_0.dvm did not match. expected checksum is 319180500 and actual is checksum 1500428316. expected length is 327 and actual length is 327
   [junit4]   2> 38539 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0_Asserting_0.dvm did not match. expected checksum is 3233289732 and actual is checksum 1024350135. expected length is 311 and actual length is 311
   [junit4]   2> 38549 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0.kdm did not match. expected checksum is 3797685714 and actual is checksum 2512835253. expected length is 143 and actual length is 143
   [junit4]   2> 38554 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0.kdm did not match. expected checksum is 2152808532 and actual is checksum 766278688. expected length is 143 and actual length is 143
   [junit4]   2> 38554 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0.si did not match. expected checksum is 2361911536 and actual is checksum 1663792527. expected length is 908 and actual length is 909
   [junit4]   2> 38567 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0.si did not match. expected checksum is 1688215311 and actual is checksum 3519575193. expected length is 888 and actual length is 928
   [junit4]   2> 38568 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0.kdi did not match. expected checksum is 419132165 and actual is checksum 1772937031. expected length is 68 and actual length is 68
   [junit4]   2> 38572 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0.kdi did not match. expected checksum is 834692742 and actual is checksum 4031691078. expected length is 68 and actual length is 68
   [junit4]   2> 38573 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0.fdm did not match. expected checksum is 2107868927 and actual is checksum 1444348952. expected length is 158 and actual length is 158
   [junit4]   2> 38582 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0_Lucene84_0.doc did not match. expected checksum is 1649391624 and actual is checksum 4254415766. expected length is 78 and actual length is 78
   [junit4]   2> 38582 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0_Lucene84_0.doc did not match. expected checksum is 195575376 and actual is checksum 2257303925. expected length is 78 and actual length is 78
   [junit4]   2> 38587 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0.fdm did not match. expected checksum is 1172096121 and actual is checksum 1292308914. expected length is 158 and actual length is 158
   [junit4]   2> 38604 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0.fdx did not match. expected checksum is 2405833444 and actual is checksum 1621389786. expected length is 64 and actual length is 64
   [junit4]   2> 38605 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0.fdx did not match. expected checksum is 1840906653 and actual is checksum 2341718745. expected length is 64 and actual length is 64
   [junit4]   2> 38615 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0.fdt did not match. expected checksum is 1075978910 and actual is checksum 421024746. expected length is 1375 and actual length is 1375
   [junit4]   2> 38617 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0.fdt did not match. expected checksum is 3041504100 and actual is checksum 2679938756. expected length is 1260 and actual length is 1260
   [junit4]   2> 38620 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0_TestBloomFilteredLucenePostings_0.blm did not match. expected checksum is 3553459509 and actual is checksum 3748904408. expected length is 147 and actual length is 147
   [junit4]   2> 38622 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0_TestBloomFilteredLucenePostings_0.blm did not match. expected checksum is 3607239205 and actual is checksum 3913800060. expected length is 147 and actual length is 147
   [junit4]   2> 38625 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0_MockRandom_0.pos did not match. expected checksum is 2092121516 and actual is checksum 2493820173. expected length is 98 and actual length is 98
   [junit4]   2> 38627 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0_MockRandom_0.pos did not match. expected checksum is 3229465183 and actual is checksum 1079021833. expected length is 100 and actual length is 100
   [junit4]   2> 38634 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0_TestBloomFilteredLucenePostings_0.tim did not match. expected checksum is 2213999863 and actual is checksum 2547651824. expected length is 825 and actual length is 825
   [junit4]   2> 38639 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0_TestBloomFilteredLucenePostings_0.tip did not match. expected checksum is 840981138 and actual is checksum 403202800. expected length is 95 and actual length is 95
   [junit4]   2> 38641 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0_TestBloomFilteredLucenePostings_0.tim did not match. expected checksum is 3737277784 and actual is checksum 1555798891. expected length is 901 and actual length is 901
   [junit4]   2> 38650 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0_Lucene84_0.tmd did not match. expected checksum is 2384817753 and actual is checksum 1172931119. expected length is 255 and actual length is 255
   [junit4]   2> 38652 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0_TestBloomFilteredLucenePostings_0.tip did not match. expected checksum is 3841520572 and actual is checksum 1718626902. expected length is 95 and actual length is 95
   [junit4]   2> 38656 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0_TestBloomFilteredLucenePostings_0.pos did not match. expected checksum is 2591345906 and actual is checksum 745483394. expected length is 119 and actual length is 119
   [junit4]   2> 38657 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0_TestBloomFilteredLucenePostings_0.pos did not match. expected checksum is 1000137729 and actual is checksum 3857609987. expected length is 121 and actual length is 121
   [junit4]   2> 38661 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0.kdd did not match. expected checksum is 845874470 and actual is checksum 1207223227. expected length is 156 and actual length is 156
   [junit4]   2> 38662 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0_Lucene84_0.tmd did not match. expected checksum is 3349743257 and actual is checksum 2511624117. expected length is 255 and actual length is 255
   [junit4]   2> 38671 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0_MockRandom_0.sd did not match. expected checksum is 2034560197 and actual is checksum 4111539108. expected length is 76 and actual length is 76
   [junit4]   2> 38671 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0.kdd did not match. expected checksum is 417776886 and actual is checksum 1902447148. expected length is 164 and actual length is 164
   [junit4]   2> 38677 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0_MockRandom_0.sd did not match. expected checksum is 288888867 and actual is checksum 3933220476. expected length is 76 and actual length is 76
   [junit4]   2> 38678 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0.fnm did not match. expected checksum is 2056900585 and actual is checksum 3867044565. expected length is 459 and actual length is 459
   [junit4]   2> 38689 WARN  (parallelCoreAdminExecutor-50-thread-1-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433622403 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard2 r:core_node7 x:backuprestore_testbackuprestoretooriginal_shard2_replica_n4] o.a.s.h.IndexFetcher File _0.nvm did not match. expected checksum is 3863859190 and actual is checksum 1931376520. expected length is 175 and actual length is 175
   [junit4]   2> 38691 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0.nvm did not match. expected checksum is 787203896 and actual is checksum 2204472445. expected length is 175 and actual length is 175
   [junit4]   2> 38697 WARN  (parallelCoreAdminExecutor-50-thread-2-processing-n:127.0.0.1:44747_solr x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1 e7395f1a-c6b7-41e6-9c64-e083fdbbb3b47863151433489641 RESTORECORE) [n:127.0.0.1:44747_solr c:backuprestore_testbackuprestoretooriginal s:shard1 r:core_node3 x:backuprestore_testbackuprestoretooriginal_shard1_replica_n1] o.a.s.h.IndexFetcher File _0.fnm did not match. expected checksum is 449689823 and actual is checksum 2552726480. expected length is 459 and actual length is 459
   [junit4]   2> 43070 ERROR (qtp1544817468-296) [    ] c.a.t.s.d.FileStore File can not be read
   [junit4]   2> com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input
   [junit4]   2>  at [Source: (File); line: 1, column: 0]
   [junit4]   2> 	at com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59) ~[jackson-databind-2.12.3.jar:2.12.3]
   [junit4]   2> 	at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688) ~[jackson-databind-2.12.3.jar:2.12.3]
   [junit4]   2> 	at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586) ~[jackson-databind-2.12.3.jar:2.12.3]
   [junit4]   2> 	at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3413) ~[jackson-databind-2.12.3.jar:2.12.3]
   [junit4]   2> 	at com.adobe.testing.s3mock.domain.FileStore.getS3Object(FileStore.java:510) ~[s3mock-2.1.34.jar:?]
   [junit4]   2> 	at com.adobe.testing.s3mock.FileStoreController.headObject(FileStoreController.java:257) ~[s3mock-2.1.34.jar:?]
   [junit4]   2> 	at sun.reflect.GeneratedMethodAccessor81.invoke(Unknown Source) ~[?:?]
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_291]
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_291]
   [junit4]   2> 	at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:190) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:138) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:105) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:878) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:792) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1040) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:943) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:898) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at javax.servlet.http.HttpServlet.doHead(HttpServlet.java:288) ~[javax.servlet-api-3.1.0.jar:3.1.0]
   [junit4]   2> 	at javax.servlet.http.HttpServlet.service(HttpServlet.java:704) ~[javax.servlet-api-3.1.0.jar:3.1.0]
   [junit4]   2> 	at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) ~[javax.servlet-api-3.1.0.jar:3.1.0]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:799) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1631) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:91) ~[s3mock-2.1.34.jar:?]
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:103) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:97) ~[spring-boot-actuator-2.3.12.RELEASE.jar:2.3.12.RELEASE]
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:600) ~[jetty-security-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:516) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645) [jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392) [jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) [jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) [jetty-io-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) [jetty-io-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) [jetty-io-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338) [jetty-util-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315) [jetty-util-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) [jetty-util-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) [jetty-util-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409) [jetty-util-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) [jetty-util-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) [jetty-util-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_291]
   [junit4]   2> com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input
   [junit4]   2>  at [Source: (File); line: 1, column: 0]
   [junit4]   2> 	at com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)
   [junit4]   2> 	at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688)
   [junit4]   2> 	at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586)
   [junit4]   2> 	at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3413)
   [junit4]   2> 	at com.adobe.testing.s3mock.domain.FileStore.getS3Object(FileStore.java:510)
   [junit4]   2> 	at com.adobe.testing.s3mock.FileStoreController.headObject(FileStoreController.java:257)
   [junit4]   2> 	at sun.reflect.GeneratedMethodAccessor81.invoke(Unknown Source)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:190)
   [junit4]   2> 	at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:138)
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:105)
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:878)
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:792)
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
   [junit4]   2> 	at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1040)
   [junit4]   2> 	at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:943)
   [junit4]   2> 	at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
   [junit4]   2> 	at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:898)
   [junit4]   2> 	at javax.servlet.http.HttpServlet.doHead(HttpServlet.java:288)
   [junit4]   2> 	at javax.servlet.http.HttpServlet.service(HttpServlet.java:704)
   [junit4]   2> 	at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
   [junit4]   2> 	at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:799)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1631)
   [junit4]   2> 	at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:91)
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
   [junit4]   2> 	at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:103)
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
   [junit4]   2> 	at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:97)
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
   [junit4]   2> 	at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
   [junit4]   2> 	at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:600)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:516)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)
   [junit4]   2> 	at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 46351 ERROR (qtp1544817468-299) [    ] c.a.t.s.d.FileStore File can not be read
   [junit4]   2> com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input
   [junit4]   2>  at [Source: (File); line: 1, column: 0]
   [junit4]   2> 	at com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59) ~[jackson-databind-2.12.3.jar:2.12.3]
   [junit4]   2> 	at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688) ~[jackson-databind-2.12.3.jar:2.12.3]
   [junit4]   2> 	at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586) ~[jackson-databind-2.12.3.jar:2.12.3]
   [junit4]   2> 	at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3413) ~[jackson-databind-2.12.3.jar:2.12.3]
   [junit4]   2> 	at com.adobe.testing.s3mock.domain.FileStore.getS3Object(FileStore.java:510) ~[s3mock-2.1.34.jar:?]
   [junit4]   2> 	at com.adobe.testing.s3mock.FileStoreController.headObject(FileStoreController.java:257) ~[s3mock-2.1.34.jar:?]
   [junit4]   2> 	at sun.reflect.GeneratedMethodAccessor81.invoke(Unknown Source) ~[?:?]
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_291]
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_291]
   [junit4]   2> 	at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:190) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:138) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:105) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:878) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:792) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1040) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:943) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:898) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at javax.servlet.http.HttpServlet.doHead(HttpServlet.java:288) ~[javax.servlet-api-3.1.0.jar:3.1.0]
   [junit4]   2> 	at javax.servlet.http.HttpServlet.service(HttpServlet.java:704) ~[javax.servlet-api-3.1.0.jar:3.1.0]
   [junit4]   2> 	at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) ~[javax.servlet-api-3.1.0.jar:3.1.0]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:799) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1631) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:91) ~[s3mock-2.1.34.jar:?]
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:103) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:97) ~[spring-boot-actuator-2.3.12.RELEASE.jar:2.3.12.RELEASE]
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:600) ~[jetty-security-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:516) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400) ~[jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645) [jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392) [jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) [jetty-server-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) [jetty-io-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) [jetty-io-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) [jetty-io-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338) [jetty-util-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315) [jetty-util-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) [jetty-util-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) [jetty-util-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409) [jetty-util-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) [jetty-util-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) [jetty-util-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_291]
   [junit4]   2> com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input
   [junit4]   2>  at [Source: (File); line: 1, column: 0]
   [junit4]   2> 	at com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)
   [junit4]   2> 	at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688)
   [junit4]   2> 	at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586)
   [junit4]   2> 	at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3413)
   [junit4]   2> 	at com.adobe.testing.s3mock.domain.FileStore.getS3Object(FileStore.java:510)
   [junit4]   2> 	at com.adobe.testing.s3mock.FileStoreController.headObject(FileStoreController.java:257)
   [junit4]   2> 	at sun.reflect.GeneratedMethodAccessor81.invoke(Unknown Source)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:190)
   [junit4]   2> 	at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:138)
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:105)
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:878)
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:792)
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
   [junit4]   2> 	at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1040)
   [junit4]   2> 	at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:943)
   [junit4]   2> 	at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
   [junit4]   2> 	at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:898)
   [junit4]   2> 	at javax.servlet.http.HttpServlet.doHead(HttpServlet.java:288)
   [junit4]   2> 	at javax.servlet.http.HttpServlet.service(HttpServlet.java:704)
   [junit4]   2> 	at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
   [junit4]   2> 	at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:799)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1631)
   [junit4]   2> 	at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:91)
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
   [junit4]   2> 	at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:103)
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
   [junit4]   2> 	at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:97)
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
   [junit4]   2> 	at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
   [junit4]   2> 	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
   [junit4]   2> 	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
   [junit4]   2> 	at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:600)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:516)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)
   [junit4]   2> 	at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 48081 WARN  (OverseerThreadFactory-49-thread-5-processing-n:127.0.0.1:41533_solr) [n:127.0.0.1:41533_solr c:backuprestore_testbackupincsimple_restore   ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (backuprestore_testbackupincsimple_restore) without cores.
   [junit4]   2> 54587 ERROR (qtp1544817468-39) [    ] c.a.t.s.d.FileStore File can not be read
   [junit4]   2> com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input
   [junit4]   2>  at [Source: (File); line: 1, column: 0]
   [junit4]   2> 	at com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59) ~[jackson-databind-2.12.3.jar:2.12.3]
   [junit4]   2> 	at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4688) ~[jackson-databind-2.12.3.jar:2.12.3]
   [junit4]   2> 	at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4586) ~[jackson-databind-2.12.3.jar:2.12.3]
   [junit4]   2> 	at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3413) ~[jackson-databind-2.12.3.jar:2.12.3]
   [junit4]   2> 	at com.adobe.testing.s3mock.domain.FileStore.getS3Object(FileStore.java:510) ~[s3mock-2.1.34.jar:?]
   [junit4]   2> 	at com.adobe.testing.s3mock.FileStoreController.headObject(FileStoreController.java:257) ~[s3mock-2.1.34.jar:?]
   [junit4]   2> 	at sun.reflect.GeneratedMethodAccessor81.invoke(Unknown Source) ~[?:?]
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_291]
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_291]
   [junit4]   2> 	at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:190) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:138) ~[spring-web-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:105) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:878) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:792) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1040) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:943) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:898) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at javax.servlet.http.HttpServlet.doHead(HttpServlet.java:288) ~[javax.servlet-api-3.1.0.jar:3.1.0]
   [junit4]   2> 	at javax.servlet.http.HttpServlet.service(HttpServlet.java:704) ~[javax.servlet-api-3.1.0.jar:3.1.0]
   [junit4]   2> 	at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883) ~[spring-webmvc-5.2.15.RELEASE.jar:5.2.15.RELEASE]
   [junit4]   2> 	at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) ~[javax.servlet-api-3.1.0.jar:3.1.0]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:799) ~[jetty-servlet-9.4.44.v20210927.jar:9.4.44.v20210927]
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1631) ~[jetty-ser

[...truncated too long message...]

 2> 	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:555)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:410)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:164)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)
   [junit4]   2> 	at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=S3IncrementalBackupTest -Dtests.method=testBackupIncremental -Dtests.seed=C1159D09C0742EC4 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar-SY -Dtests.timezone=Europe/Saratov -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   3.44s J2 | S3IncrementalBackupTest.testBackupIncremental <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:44747/solr: Could not backup all shards
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C1159D09C0742EC4:C6D032C2FC9C10B9]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:681)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:369)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:297)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1171)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:934)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:866)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:231)
   [junit4]    > 	at org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest$IncrementalBackupVerifier.backupThenWait(AbstractIncrementalBackupTest.java:477)
   [junit4]    > 	at org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest$IncrementalBackupVerifier.incrementalBackupThenVerify(AbstractIncrementalBackupTest.java:487)
   [junit4]    > 	at org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest.backupRestoreThenCheck(AbstractIncrementalBackupTest.java:373)
   [junit4]    > 	at org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest.testBackupIncremental(AbstractIncrementalBackupTest.java:244)
   [junit4]    > 	at com.adobe.testing.s3mock.junit4.S3MockRule$1.evaluate(S3MockRule.java:66)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 55591 WARN  (jetty-launcher-9-thread-2-SendThread(127.0.0.1:43063)) [    ] o.a.z.ClientCnxn An exception was thrown while closing send thread for session 0x101d4adee7c0006.
   [junit4]   2> org.apache.zookeeper.ClientCnxn$EndOfStreamException: Unable to read additional data from server sessionid 0x101d4adee7c0006, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:77) ~[zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:350) ~[zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1275) [zookeeper-3.6.2.jar:3.6.2]
   [junit4]   2> 55694 INFO  (jetty-closer-177-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@4de0da61{SSL, (ssl, http/1.1)}{127.0.0.1:0}
   [junit4]   2> 55694 INFO  (jetty-closer-177-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@14848477{/solr,null,STOPPED}
   [junit4]   2> 55694 INFO  (jetty-closer-177-thread-2) [    ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 55793 INFO  (jetty-closer-177-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@16250f41{SSL, (ssl, http/1.1)}{127.0.0.1:0}
   [junit4]   2> 55794 INFO  (jetty-closer-177-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@25f5ee91{/solr,null,STOPPED}
   [junit4]   2> 55794 INFO  (jetty-closer-177-thread-1) [    ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 55999 WARN  (ZkTestServer Run Thread) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	7	/solr/collections/backuprestore_testbackupinc/terms/shard2
   [junit4]   2> 	7	/solr/collections/backuprestore_testbackupinc/terms/shard1
   [junit4]   2> 	7	/solr/collections/backuprestore_testbackuprestoretooriginal/terms/shard2
   [junit4]   2> 	7	/solr/collections/backuprestore_testbackuprestoretooriginal/terms/shard1
   [junit4]   2> 	3	/solr/collections/backuprestore_testbackupincsimple_restore/terms/shard2
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	3	/solr/collections/backuprestore_testbackupincsimple_restore/terms/shard1
   [junit4]   2> 	3	/solr/collections/backuprestore_testbackupincsimple/terms/shard2
   [junit4]   2> 	3	/solr/collections/backuprestore_testbackupincsimple/terms/shard1
   [junit4]   2> 	2	/solr/packages.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 	2	/solr/collections/backuprestore_testbackupincsimple/collectionprops.json
   [junit4]   2> 	2	/solr/configs/conf1
   [junit4]   2> 	2	/solr/collections/backuprestore_testbackuprestoretooriginal/collectionprops.json
   [junit4]   2> 	2	/solr/collections/backuprestore_testbackupinc/collectionprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	50	/solr/collections/backuprestore_testbackupincsimple_restore/state.json
   [junit4]   2> 	40	/solr/collections/backuprestore_testbackupinc/state.json
   [junit4]   2> 	39	/solr/collections/backuprestore_testbackuprestoretooriginal/state.json
   [junit4]   2> 	18	/solr/collections/backuprestore_testbackupincsimple/state.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	87	/solr/overseer/queue
   [junit4]   2> 	23	/solr/overseer/collection-queue-work
   [junit4]   2> 	15	/solr/collections
   [junit4]   2> 	10	/solr/collections/backuprestore_testbackuprestoretooriginal/state.json
   [junit4]   2> 	10	/solr/collections/backuprestore_testbackupinc/state.json
   [junit4]   2> 	8	/solr/collections/backuprestore_testbackupincsimple_restore/state.json
   [junit4]   2> 	8	/solr/live_nodes
   [junit4]   2> 	8	/solr/collections/backuprestore_testbackupincsimple/state.json
   [junit4]   2> 
   [junit4]   2> 56010 INFO  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker) [    ] o.e.j.s.AbstractConnector Stopped SslValidatingServerConnector@7ed39dc4{SSL, (ssl, http/1.1)}{0.0.0.0:0}
   [junit4]   2> 56011 INFO  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@26f3bb57{HTTP/1.1, (http/1.1)}{0.0.0.0:0}
   [junit4]   2> 56011 INFO  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker) [    ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 56011 INFO  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker) [    ] o.e.j.s.h.C.application Destroying Spring FrameworkServlet 'dispatcherServlet'
   [junit4]   2> 56012 INFO  (SUITE-S3IncrementalBackupTest-seed#[C1159D09C0742EC4]-worker) [    ] o.e.j.s.h.ContextHandler Stopped o.s.b.w.e.j.JettyEmbeddedWebAppContext@38a93c5e{application,/,[file:///home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr/build/contrib/solr-s3-repository/test/J2/temp/jetty-docbase.0.6258932750794238146/],STOPPED}
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-Tests-8.10/solr/build/contrib/solr-s3-repository/test/J2/temp/solr.s3.S3IncrementalBackupTest_C1159D09C0742EC4-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene87): {shard_s=PostingsFormat(name=MockRandom), _root_=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene84)), id=Lucene84}, docValues:{_version_=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=625, maxMBSortInHeap=5.224847883571704, sim=Asserting(RandomSimilarity(queryNorm=false): {}), locale=ar-SY, timezone=Europe/Saratov
   [junit4]   2> NOTE: Linux 4.15.0-147-generic amd64/Oracle Corporation 1.8.0_291 (64-bit)/cpus=4,threads=1,free=70454472,total=466616320
   [junit4]   2> NOTE: All tests run in this JVM: [S3ReadWriteTest, S3IncrementalBackupTest]
   [junit4] Completed [7/7 (1!)] on J2 in 37.50s, 3 tests, 1 error <<< FAILURES!

[...truncated 44700 lines...]