You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2020/01/27 02:08:28 UTC
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk-12.0.2) - Build #
5575 - Still Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/5575/
Java: 64bit/jdk-12.0.2 -XX:-UseCompressedOops -XX:+UseSerialGC
2 tests failed.
FAILED: org.apache.solr.cloud.DeleteReplicaTest.deleteReplicaAndVerifyDirectoryCleanup
Error Message:
No registered leader was found after waiting for 4000ms , collection: deletereplica_test slice: shard1 saw state=DocCollection(deletereplica_test//collections/deletereplica_test/state.json/7)={ "pullReplicas":"0", "replicationFactor":"2", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{"core_node4":{ "core":"deletereplica_test_shard1_replica_n2", "base_url":"http://127.0.0.1:62441/solr", "node_name":"127.0.0.1:62441_solr", "state":"active", "type":"NRT", "force_set_state":"false"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"2", "tlogReplicas":"0"} with live_nodes=[127.0.0.1:62439_solr, 127.0.0.1:62441_solr, 127.0.0.1:62438_solr, 127.0.0.1:62440_solr]
Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 4000ms , collection: deletereplica_test slice: shard1 saw state=DocCollection(deletereplica_test//collections/deletereplica_test/state.json/7)={
"pullReplicas":"0",
"replicationFactor":"2",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{"core_node4":{
"core":"deletereplica_test_shard1_replica_n2",
"base_url":"http://127.0.0.1:62441/solr",
"node_name":"127.0.0.1:62441_solr",
"state":"active",
"type":"NRT",
"force_set_state":"false"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"2",
"tlogReplicas":"0"} with live_nodes=[127.0.0.1:62439_solr, 127.0.0.1:62441_solr, 127.0.0.1:62438_solr, 127.0.0.1:62440_solr]
at __randomizedtesting.SeedInfo.seed([B830D02E9D3F258C:C8D0549419708C87]:0)
at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:979)
at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:956)
at org.apache.solr.cloud.DeleteReplicaTest.deleteReplicaAndVerifyDirectoryCleanup(DeleteReplicaTest.java:164)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:567)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
at java.base/java.lang.Thread.run(Thread.java:835)
FAILED: org.apache.solr.cloud.SystemCollectionCompatTest.testBackCompat
Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:53311/solr/.system]
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:53311/solr/.system]
at __randomizedtesting.SeedInfo.seed([B830D02E9D3F258C:C8C57387FDF78CFA]:0)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:345)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1143)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:906)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:838)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
at org.apache.solr.cloud.SystemCollectionCompatTest.setupSystemCollection(SystemCollectionCompatTest.java:118)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:567)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:976)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
at java.base/java.lang.Thread.run(Thread.java:835)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:53311/solr/.system
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:676)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:265)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
... 43 more
Caused by: java.net.SocketTimeoutException: Read timed out
at java.base/java.net.SocketInputStream.socketRead0(Native Method)
at java.base/java.net.SocketInputStream.socketRead(SocketInputStream.java:115)
at java.base/java.net.SocketInputStream.read(SocketInputStream.java:168)
at java.base/java.net.SocketInputStream.read(SocketInputStream.java:140)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:564)
... 47 more
Build Log:
[...truncated 13765 lines...]
[junit4] Suite: org.apache.solr.cloud.SystemCollectionCompatTest
[junit4] 2> 402362 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 402362 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.SolrTestCaseJ4 Created dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J2/temp/solr.cloud.SystemCollectionCompatTest_B830D02E9D3F258C-001/data-dir-28-001
[junit4] 2> 402363 WARN (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=6 numCloses=6
[junit4] 2> 402363 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 402364 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason="", value=0.0/0.0, ssl=0.0/0.0, clientAuth=0.0/0.0) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 402364 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J2/temp/solr.cloud.SystemCollectionCompatTest_B830D02E9D3F258C-001/tempDir-001
[junit4] 2> 402364 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 402365 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 402365 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 402470 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:53304
[junit4] 2> 402470 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:53304
[junit4] 2> 402470 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:53304
[junit4] 2> 402470 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 53304
[junit4] 2> 402474 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 402479 INFO (zkConnectionManagerCallback-2595-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 402479 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 402484 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 402488 INFO (zkConnectionManagerCallback-2597-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 402488 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 402490 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 402496 INFO (zkConnectionManagerCallback-2599-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 402496 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 402608 WARN (jetty-launcher-2600-thread-1) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 402608 WARN (jetty-launcher-2600-thread-2) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 402608 INFO (jetty-launcher-2600-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 402608 INFO (jetty-launcher-2600-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 402608 INFO (jetty-launcher-2600-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 402608 INFO (jetty-launcher-2600-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
[junit4] 2> 402608 INFO (jetty-launcher-2600-thread-2) [ ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 12.0.2+10
[junit4] 2> 402608 INFO (jetty-launcher-2600-thread-1) [ ] o.e.j.s.Server jetty-9.4.24.v20191120; built: 2019-11-20T21:37:49.771Z; git: 363d5f2df3a8a28de40604320230664b9c793c16; jvm 12.0.2+10
[junit4] 2> 402612 INFO (jetty-launcher-2600-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 402613 INFO (jetty-launcher-2600-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 402613 INFO (jetty-launcher-2600-thread-2) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 402613 INFO (jetty-launcher-2600-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@40fe25ad{/solr,null,AVAILABLE}
[junit4] 2> 402614 INFO (jetty-launcher-2600-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 402614 INFO (jetty-launcher-2600-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 402614 INFO (jetty-launcher-2600-thread-1) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 402614 INFO (jetty-launcher-2600-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@60adb77f{/solr,null,AVAILABLE}
[junit4] 2> 402614 INFO (jetty-launcher-2600-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6ca131b0{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:53311}
[junit4] 2> 402614 INFO (jetty-launcher-2600-thread-2) [ ] o.e.j.s.Server Started @402694ms
[junit4] 2> 402614 INFO (jetty-launcher-2600-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=53311}
[junit4] 2> 402615 ERROR (jetty-launcher-2600-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 402615 INFO (jetty-launcher-2600-thread-2) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 402615 INFO (jetty-launcher-2600-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
[junit4] 2> 402615 INFO (jetty-launcher-2600-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 402615 INFO (jetty-launcher-2600-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 402615 INFO (jetty-launcher-2600-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-01-27T01:16:40.094450Z
[junit4] 2> 402615 INFO (jetty-launcher-2600-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@624a2c59{HTTP/1.1,[http/1.1, h2c]}{127.0.0.1:53312}
[junit4] 2> 402615 INFO (jetty-launcher-2600-thread-1) [ ] o.e.j.s.Server Started @402695ms
[junit4] 2> 402615 INFO (jetty-launcher-2600-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=53312}
[junit4] 2> 402617 ERROR (jetty-launcher-2600-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 402617 INFO (jetty-launcher-2600-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 402617 INFO (jetty-launcher-2600-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 9.0.0
[junit4] 2> 402617 INFO (jetty-launcher-2600-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 402617 INFO (jetty-launcher-2600-thread-2) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 402617 INFO (jetty-launcher-2600-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 402617 INFO (jetty-launcher-2600-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-01-27T01:16:40.096346Z
[junit4] 2> 402620 INFO (jetty-launcher-2600-thread-1) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 402633 INFO (zkConnectionManagerCallback-2602-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 402633 INFO (jetty-launcher-2600-thread-2) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 402635 INFO (jetty-launcher-2600-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 402635 INFO (zkConnectionManagerCallback-2604-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 402635 INFO (jetty-launcher-2600-thread-1) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 402637 INFO (jetty-launcher-2600-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 402893 INFO (jetty-launcher-2600-thread-2) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 402897 WARN (jetty-launcher-2600-thread-2) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@4fb4f6ea[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 402897 WARN (jetty-launcher-2600-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@4fb4f6ea[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 402904 WARN (jetty-launcher-2600-thread-2) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@4525a2da[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 402904 WARN (jetty-launcher-2600-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@4525a2da[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 402905 INFO (jetty-launcher-2600-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53304/solr
[junit4] 2> 402910 INFO (jetty-launcher-2600-thread-2) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 402919 INFO (zkConnectionManagerCallback-2612-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 402919 INFO (jetty-launcher-2600-thread-2) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 403042 INFO (jetty-launcher-2600-thread-2) [n:127.0.0.1:53311_solr ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 403045 INFO (zkConnectionManagerCallback-2614-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 403047 INFO (jetty-launcher-2600-thread-2) [n:127.0.0.1:53311_solr ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 403117 INFO (jetty-launcher-2600-thread-1) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
[junit4] 2> 403120 WARN (jetty-launcher-2600-thread-1) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@29e6fd9c[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 403121 WARN (jetty-launcher-2600-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@29e6fd9c[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 403128 WARN (jetty-launcher-2600-thread-1) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@2c93c30e[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 403128 WARN (jetty-launcher-2600-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@2c93c30e[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 403129 INFO (jetty-launcher-2600-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53304/solr
[junit4] 2> 403133 INFO (jetty-launcher-2600-thread-1) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 403137 INFO (zkConnectionManagerCallback-2622-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 403137 INFO (jetty-launcher-2600-thread-1) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 403250 INFO (jetty-launcher-2600-thread-1) [n:127.0.0.1:53312_solr ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 403259 INFO (zkConnectionManagerCallback-2624-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 403259 INFO (jetty-launcher-2600-thread-1) [n:127.0.0.1:53312_solr ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 403340 INFO (jetty-launcher-2600-thread-1) [n:127.0.0.1:53312_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:53312_solr
[junit4] 2> 403343 INFO (jetty-launcher-2600-thread-1) [n:127.0.0.1:53312_solr ] o.a.s.c.Overseer Overseer (id=72075166147674120-127.0.0.1:53312_solr-n_0000000000) starting
[junit4] 2> 403361 INFO (OverseerStateUpdate-72075166147674120-127.0.0.1:53312_solr-n_0000000000) [n:127.0.0.1:53312_solr ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:53312_solr
[junit4] 2> 403363 INFO (jetty-launcher-2600-thread-1) [n:127.0.0.1:53312_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53312_solr
[junit4] 2> 403369 INFO (zkCallback-2623-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 403377 INFO (jetty-launcher-2600-thread-1) [n:127.0.0.1:53312_solr ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 403378 WARN (jetty-launcher-2600-thread-1) [n:127.0.0.1:53312_solr ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 403423 INFO (jetty-launcher-2600-thread-1) [n:127.0.0.1:53312_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 403512 INFO (jetty-launcher-2600-thread-1) [n:127.0.0.1:53312_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53312.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1181525f
[junit4] 2> 403549 INFO (jetty-launcher-2600-thread-1) [n:127.0.0.1:53312_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53312.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1181525f
[junit4] 2> 403549 INFO (jetty-launcher-2600-thread-1) [n:127.0.0.1:53312_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53312.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1181525f
[junit4] 2> 403563 INFO (jetty-launcher-2600-thread-1) [n:127.0.0.1:53312_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J2/temp/solr.cloud.SystemCollectionCompatTest_B830D02E9D3F258C-001/tempDir-001/node1/.
[junit4] 2> 403656 INFO (jetty-launcher-2600-thread-2) [n:127.0.0.1:53311_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 403666 INFO (jetty-launcher-2600-thread-2) [n:127.0.0.1:53311_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:53311_solr as DOWN
[junit4] 2> 403668 INFO (jetty-launcher-2600-thread-2) [n:127.0.0.1:53311_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 403668 INFO (jetty-launcher-2600-thread-2) [n:127.0.0.1:53311_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53311_solr
[junit4] 2> 403671 INFO (zkCallback-2623-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 403671 INFO (zkCallback-2613-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 403675 INFO (jetty-launcher-2600-thread-2) [n:127.0.0.1:53311_solr ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 403675 WARN (jetty-launcher-2600-thread-2) [n:127.0.0.1:53311_solr ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 403710 INFO (jetty-launcher-2600-thread-2) [n:127.0.0.1:53311_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 403739 INFO (jetty-launcher-2600-thread-2) [n:127.0.0.1:53311_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53311.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1181525f
[junit4] 2> 403765 INFO (jetty-launcher-2600-thread-2) [n:127.0.0.1:53311_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53311.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1181525f
[junit4] 2> 403766 INFO (jetty-launcher-2600-thread-2) [n:127.0.0.1:53311_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53311.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1181525f
[junit4] 2> 403768 INFO (jetty-launcher-2600-thread-2) [n:127.0.0.1:53311_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J2/temp/solr.cloud.SystemCollectionCompatTest_B830D02E9D3F258C-001/tempDir-001/node2/.
[junit4] 2> 403805 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=2
[junit4] 2> 403806 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 403811 INFO (zkConnectionManagerCallback-2635-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 403811 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 403817 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 403827 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53304/solr ready
[junit4] 2> 403894 INFO (TEST-SystemCollectionCompatTest.testBackCompat-seed#[B830D02E9D3F258C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testBackCompat
[junit4] 2> 403897 INFO (TEST-SystemCollectionCompatTest.testBackCompat-seed#[B830D02E9D3F258C]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 403904 INFO (zkConnectionManagerCallback-2640-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 403905 INFO (TEST-SystemCollectionCompatTest.testBackCompat-seed#[B830D02E9D3F258C]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 403913 INFO (TEST-SystemCollectionCompatTest.testBackCompat-seed#[B830D02E9D3F258C]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 403920 INFO (TEST-SystemCollectionCompatTest.testBackCompat-seed#[B830D02E9D3F258C]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53304/solr ready
[junit4] 2> 403924 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :overseerstatus with params action=OVERSEERSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 403940 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=OVERSEERSTATUS&wt=javabin&version=2} status=0 QTime=15
[junit4] 2> 403942 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params maxShardsPerNode=2&name=.system&nrtReplicas=2&action=CREATE&numShards=1&createNodeSet=127.0.0.1:53311_solr&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 403959 INFO (OverseerThreadFactory-1940-thread-2-processing-n:127.0.0.1:53312_solr) [n:127.0.0.1:53312_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection .system
[junit4] 2> 403959 INFO (OverseerCollectionConfigSetProcessor-72075166147674120-127.0.0.1:53312_solr-n_0000000000) [n:127.0.0.1:53312_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 404071 WARN (OverseerThreadFactory-1940-thread-2-processing-n:127.0.0.1:53312_solr) [n:127.0.0.1:53312_solr ] o.a.s.c.a.c.CreateCollectionCmd Specified number of replicas of 2 on collection .system is higher than the number of Solr instances currently live or live and part of your createNodeSet(1). It's unusual to run two replica of the same slice on the same Solr-instance.
[junit4] 2> 404080 INFO (OverseerStateUpdate-72075166147674120-127.0.0.1:53312_solr-n_0000000000) [n:127.0.0.1:53312_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":".system",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":".system_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:53311/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 404096 INFO (OverseerStateUpdate-72075166147674120-127.0.0.1:53312_solr-n_0000000000) [n:127.0.0.1:53312_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":".system",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":".system_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:53311/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 404314 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr x:.system_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=.system&newCollection=true&name=.system_shard1_replica_n1&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 404315 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=.system&newCollection=true&name=.system_shard1_replica_n2&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 405354 WARN (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.SolrConfig You should not use LATEST as luceneMatchVersion property: if you use this setting, and then Solr upgrades to a newer release of Lucene, sizable changes may happen. If precise back compatibility is important then you should instead explicitly specify an actual Lucene version.
[junit4] 2> 405354 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 405356 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 405364 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
[junit4] 2> 405365 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
[junit4] 2> 405369 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.IndexSchema [.system_shard1_replica_n2] Schema name=_system collection or core
[junit4] 2> 405369 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.IndexSchema [.system_shard1_replica_n1] Schema name=_system collection or core
[junit4] 2> 405387 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
[junit4] 2> 405387 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
[junit4] 2> 405394 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Created and persisted managed schema znode at /configs/.system/managed-schema
[junit4] 2> 405394 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ManagedIndexSchema Managed schema znode at /configs/.system/managed-schema already exists - no need to create it
[junit4] 2> 405398 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.CoreContainer Creating SolrCore '.system_shard1_replica_n2' using configuration from configset .system, trusted=true
[junit4] 2> 405398 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53311.solr.core..system.shard1.replica_n2' (registry 'solr.core..system.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1181525f
[junit4] 2> 405398 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.SolrCore [[.system_shard1_replica_n2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J2/temp/solr.cloud.SystemCollectionCompatTest_B830D02E9D3F258C-001/tempDir-001/node2/.system_shard1_replica_n2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J2/temp/solr.cloud.SystemCollectionCompatTest_B830D02E9D3F258C-001/tempDir-001/node2/./.system_shard1_replica_n2/data/]
[junit4] 2> 405404 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ManagedIndexSchemaFactory After upgrading to managed schema in ZooKeeper, renamed the non-managed schema /configs/.system/schema.xml to /configs/.system/schema.xml.bak
[junit4] 2> 405404 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore '.system_shard1_replica_n1' using configuration from configset .system, trusted=true
[junit4] 2> 405405 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53311.solr.core..system.shard1.replica_n1' (registry 'solr.core..system.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1181525f
[junit4] 2> 405405 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.SolrCore [[.system_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J2/temp/solr.cloud.SystemCollectionCompatTest_B830D02E9D3F258C-001/tempDir-001/node2/.system_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J2/temp/solr.cloud.SystemCollectionCompatTest_B830D02E9D3F258C-001/tempDir-001/node2/./.system_shard1_replica_n1/data/]
[junit4] 2> 405468 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 405468 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 405471 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.u.CommitTracker Hard AutoCommit: if 1 uncommitted docs;
[junit4] 2> 405471 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 405473 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@58a4aab8[.system_shard1_replica_n2] main]
[junit4] 2> 405475 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 405475 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 405476 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: if 1 uncommitted docs;
[junit4] 2> 405476 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 405477 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/.system
[junit4] 2> 405477 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/.system
[junit4] 2> 405477 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@63f853c6[.system_shard1_replica_n1] main]
[junit4] 2> 405481 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/.system
[junit4] 2> 405481 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/.system/managed-schema
[junit4] 2> 405481 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/.system
[junit4] 2> 405481 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/.system/managed-schema
[junit4] 2> 405483 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 405484 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 0 from ZooKeeper
[junit4] 2> 405484 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 405484 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1656842148079730688
[junit4] 2> 405486 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.IndexSchema [.system_shard1_replica_n1] Schema name=_system collection or core
[junit4] 2> 405487 INFO (searcherExecutor-1949-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.SolrCore [.system_shard1_replica_n2] Registered new searcher Searcher@58a4aab8[.system_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 405511 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
[junit4] 2> 405511 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 26 ms
[junit4] 2> 405511 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 405512 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1656842148109090816
[junit4] 2> 405514 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/.system/terms/shard1 to Terms{values={core_node4=0}, version=0}
[junit4] 2> 405514 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/.system/leaders/shard1
[junit4] 2> 405515 INFO (searcherExecutor-1950-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n1 c:.system s:shard1 r:core_node3) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.SolrCore [.system_shard1_replica_n1] Registered new searcher Searcher@63f853c6[.system_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 405518 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/.system/terms/shard1 to Terms{values={core_node3=0, core_node4=0}, version=1}
[junit4] 2> 405518 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/.system/leaders/shard1
[junit4] 2> 405530 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 405530 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 405530 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53311/solr/.system_shard1_replica_n2/
[junit4] 2> 405531 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.u.PeerSync PeerSync: core=.system_shard1_replica_n2 url=http://127.0.0.1:53311/solr START replicas=[http://127.0.0.1:53311/solr/.system_shard1_replica_n1/] nUpdates=100
[junit4] 2> 405531 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.u.PeerSync PeerSync: core=.system_shard1_replica_n2 url=http://127.0.0.1:53311/solr DONE. We have no versions. sync failed.
[junit4] 2> 405534 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.S.Request [.system_shard1_replica_n1] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 405534 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 405534 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 405534 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/.system/leaders/shard1/leader after winning as /collections/.system/leader_elect/shard1/election/72075166147674118-core_node4-n_0000000000
[junit4] 2> 405539 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53311/solr/.system_shard1_replica_n2/ shard1
[junit4] 2> 405651 INFO (zkCallback-2613-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [2])
[junit4] 2> 405652 INFO (zkCallback-2613-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [2])
[junit4] 2> 405653 INFO (zkCallback-2613-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [2])
[junit4] 2> 405655 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 405659 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=.system&newCollection=true&name=.system_shard1_replica_n2&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1344
[junit4] 2> 405760 INFO (zkCallback-2613-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [2])
[junit4] 2> 405760 INFO (zkCallback-2613-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [2])
[junit4] 2> 405760 INFO (zkCallback-2613-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [2])
[junit4] 2> 405760 INFO (zkCallback-2613-thread-4) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [2])
[junit4] 2> 406535 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=.system&newCollection=true&name=.system_shard1_replica_n1&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2220
[junit4] 2> 406539 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 406649 INFO (zkCallback-2613-thread-4) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [2])
[junit4] 2> 406649 INFO (zkCallback-2613-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [2])
[junit4] 2> 406649 INFO (zkCallback-2613-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [2])
[junit4] 2> 406649 INFO (zkCallback-2613-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [2])
[junit4] 2> 406651 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={maxShardsPerNode=2&name=.system&nrtReplicas=2&action=CREATE&numShards=1&createNodeSet=127.0.0.1:53311_solr&wt=javabin&version=2} status=0 QTime=2708
[junit4] 2> 406651 INFO (TEST-SystemCollectionCompatTest.testBackCompat-seed#[B830D02E9D3F258C]) [ ] o.a.s.c.MiniSolrCloudCluster waitForActiveCollection: .system
[junit4] 2> 406659 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/.system/terms/shard1 to Terms{values={core_node3=1, core_node4=1}, version=2}
[junit4] 2> 406663 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [.system_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53311/solr/.system_shard1_replica_n2/&wt=javabin&version=2}{add=[16fe49280e4Taqhercg0oubevok6mgqu3hp2n (1656842149308661760)]} 0 1
[junit4] 2> 406664 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.u.p.LogUpdateProcessorFactory [.system_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[16fe49280e4Taqhercg0oubevok6mgqu3hp2n (1656842149308661760)]} 0 9
[junit4] 2> 406664 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1656842149317050368,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 406664 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@497ffc0d commitCommandVersion:1656842149317050368
[junit4] 2> 406665 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1656842149318098944,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 406665 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@74ebbd5 commitCommandVersion:1656842149318098944
[junit4] 2> 406666 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@7673a5fc[.system_shard1_replica_n2] main]
[junit4] 2> 406666 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@418086b[.system_shard1_replica_n1] main]
[junit4] 2> 406667 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 406667 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 406668 INFO (searcherExecutor-1949-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.SolrCore [.system_shard1_replica_n2] Registered new searcher Searcher@7673a5fc[.system_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(9.0.0):C1:[diagnostics={java.version=12.0.2, timestamp=1580087804144, java.vendor=AdoptOpenJDK, os=Mac OS X, os.version=10.14.6, java.runtime.version=12.0.2+10, os.arch=x86_64, source=flush, lucene.version=9.0.0, java.vm.version=12.0.2+10}]:[attributes={Lucene50StoredFieldsFormat.mode=BEST_SPEED}])))}
[junit4] 2> 406668 INFO (searcherExecutor-1950-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n1 c:.system s:shard1 r:core_node3) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.SolrCore [.system_shard1_replica_n1] Registered new searcher Searcher@418086b[.system_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(9.0.0):C1:[diagnostics={java.version=12.0.2, timestamp=1580087804145, java.vendor=AdoptOpenJDK, os=Mac OS X, os.version=10.14.6, java.runtime.version=12.0.2+10, os.arch=x86_64, source=flush, lucene.version=9.0.0, java.vm.version=12.0.2+10}]:[attributes={Lucene50StoredFieldsFormat.mode=BEST_SPEED}])))}
[junit4] 2> 406669 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [.system_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53311/solr/.system_shard1_replica_n2/&commit_end_point=replicas&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 4
[junit4] 2> 406669 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.u.p.LogUpdateProcessorFactory [.system_shard1_replica_n2] webapp=/solr path=/update params={_stateVer_=.system:5&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 5
[junit4] 2> 406672 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={action=STATUS&indexInfo=true&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 406673 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={action=STATUS&indexInfo=true&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 406675 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.S.Request [.system_shard1_replica_n2] webapp=/solr path=/schema params={wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 406684 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.SchemaManager managed schema loaded . version : 1
[junit4] 2> 406685 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.IndexSchema [.system_shard1_replica_n2] Schema name=_system collection or core
[junit4] 2> 406687 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
[junit4] 2> 406693 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.ZkController Persisted config data to node /configs/.system/schema.xml.bak
[junit4] 2> 406695 INFO (Thread-1131) [ ] o.a.s.c.SolrCore config update listener called for core .system_shard1_replica_n1
[junit4] 2> 406698 INFO (Thread-1131) [ ] o.a.s.c.SolrCore config update listener called for core .system_shard1_replica_n2
[junit4] 2> 406700 INFO (Thread-1131) [ ] o.a.s.c.SolrCore core reload .system_shard1_replica_n2
[junit4] 2> 406705 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 406712 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.IndexSchema [.system_shard1_replica_n2] Schema name=_system collection or core
[junit4] 2> 406714 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
[junit4] 2> 406715 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.CoreContainer Reloading SolrCore '.system_shard1_replica_n2' using configuration from configset .system
[junit4] 2> 406752 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53311.solr.core..system.shard1.replica_n2' (registry 'solr.core..system.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1181525f
[junit4] 2> 406752 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.SolrCore [[.system_shard1_replica_n2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J2/temp/solr.cloud.SystemCollectionCompatTest_B830D02E9D3F258C-001/tempDir-001/node2/.system_shard1_replica_n2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J2/temp/solr.cloud.SystemCollectionCompatTest_B830D02E9D3F258C-001/tempDir-001/node2/./.system_shard1_replica_n2/data/]
[junit4] 2> 406822 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.u.CommitTracker Hard AutoCommit: if 1 uncommitted docs;
[junit4] 2> 406822 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 406824 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@383e5a80[.system_shard1_replica_n2] main]
[junit4] 2> 406827 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/.system
[junit4] 2> 406827 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/.system
[junit4] 2> 406827 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/.system/managed-schema
[junit4] 2> 406828 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 406828 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 406830 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 406831 INFO (searcherExecutor-1959-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.SolrCore [.system_shard1_replica_n2] Registered new searcher Searcher@383e5a80[.system_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(9.0.0):C1:[diagnostics={java.version=12.0.2, timestamp=1580087804144, java.vendor=AdoptOpenJDK, os=Mac OS X, os.version=10.14.6, java.runtime.version=12.0.2+10, os.arch=x86_64, source=flush, lucene.version=9.0.0, java.vm.version=12.0.2+10}]:[attributes={Lucene50StoredFieldsFormat.mode=BEST_SPEED}])))}
[junit4] 2> 406832 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.SolrIndexSearcher Opening [Searcher@369348ed[.system_shard1_replica_n2] main]
[junit4] 2> 406833 INFO (searcherExecutor-1959-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.SolrCore [.system_shard1_replica_n2] Registered new searcher Searcher@369348ed[.system_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(9.0.0):C1:[diagnostics={java.version=12.0.2, timestamp=1580087804144, os=Mac OS X, java.vendor=AdoptOpenJDK, os.version=10.14.6, java.runtime.version=12.0.2+10, os.arch=x86_64, source=flush, lucene.version=9.0.0, java.vm.version=12.0.2+10}]:[attributes={Lucene50StoredFieldsFormat.mode=BEST_SPEED}])))}
[junit4] 2> 406834 INFO (qtp1650901900-6989) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Waiting up to 599 secs for 1 replicas to apply schema update version 2 for collection .system
[junit4] 2> 406839 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.h.SchemaHandler REFRESHING SCHEMA (refreshIfBelowVersion=2, currentVersion=0) before returning version!
[junit4] 2> 406840 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 406840 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.S.Request [.system_shard1_replica_n1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=2&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 406841 ERROR (managedSchemaExecutor-1963-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Replica http://127.0.0.1:53311/solr/.system_shard1_replica_n1/ returned schema version 0 and has not applied schema version 2
[junit4] 2> 406842 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.h.SchemaHandler REFRESHING SCHEMA (refreshIfBelowVersion=2, currentVersion=0) before returning version!
[junit4] 2> 406843 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 406843 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.S.Request [.system_shard1_replica_n1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=2&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 406844 INFO (Thread-1131) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
[junit4] 2> 406845 ERROR (managedSchemaExecutor-1963-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Replica http://127.0.0.1:53311/solr/.system_shard1_replica_n1/ returned schema version 0 and has not applied schema version 2
[junit4] 2> 406846 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.h.SchemaHandler REFRESHING SCHEMA (refreshIfBelowVersion=2, currentVersion=0) before returning version!
[junit4] 2> 406847 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 406847 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.S.Request [.system_shard1_replica_n1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=2&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 406847 ERROR (managedSchemaExecutor-1963-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Replica http://127.0.0.1:53311/solr/.system_shard1_replica_n1/ returned schema version 0 and has not applied schema version 2
[junit4] 2> 406848 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.h.SchemaHandler REFRESHING SCHEMA (refreshIfBelowVersion=2, currentVersion=0) before returning version!
[junit4] 2> 406849 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 406849 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.S.Request [.system_shard1_replica_n1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=2&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 406849 ERROR (managedSchemaExecutor-1963-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Replica http://127.0.0.1:53311/solr/.system_shard1_replica_n1/ returned schema version 0 and has not applied schema version 2
[junit4] 2> 406850 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.h.SchemaHandler REFRESHING SCHEMA (refreshIfBelowVersion=2, currentVersion=0) before returning version!
[junit4] 2> 406851 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 406851 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.S.Request [.system_shard1_replica_n1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=2&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 406851 ERROR (managedSchemaExecutor-1963-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Replica http://127.0.0.1:53311/solr/.system_shard1_replica_n1/ returned schema version 0 and has not applied schema version 2
[junit4] 2> 406852 INFO (Thread-1131) [ ] o.a.s.s.IndexSchema [.system_shard1_replica_n2] Schema name=_system collection or core
[junit4] 2> 406852 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.h.SchemaHandler REFRESHING SCHEMA (refreshIfBelowVersion=2, currentVersion=0) before returning version!
[junit4] 2> 406852 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 406852 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.S.Request [.system_shard1_replica_n1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=2&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 406853 ERROR (managedSchemaExecutor-1963-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Replica http://127.0.0.1:53311/solr/.system_shard1_replica_n1/ returned schema version 0 and has not applied schema version 2
[junit4] 2> 406853 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.h.SchemaHandler REFRESHING SCHEMA (refreshIfBelowVersion=2, currentVersion=0) before returning version!
[junit4] 2> 406854 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 406854 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.S.Request [.system_shard1_replica_n1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=2&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 406854 INFO (Thread-1131) [ ] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
[junit4] 2> 406854 INFO (Thread-1131) [ ] o.a.s.c.CoreContainer Reloading SolrCore '.system_shard1_replica_n2' using configuration from configset .system
[junit4] 2> 406854 ERROR (managedSchemaExecutor-1963-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Replica http://127.0.0.1:53311/solr/.system_shard1_replica_n1/ returned schema version 0 and has not applied schema version 2
[junit4] 2> 406856 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.h.SchemaHandler REFRESHING SCHEMA (refreshIfBelowVersion=2, currentVersion=0) before returning version!
[junit4] 2> 406856 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 406856 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.S.Request [.system_shard1_replica_n1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=2&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 406857 ERROR (managedSchemaExecutor-1963-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Replica http://127.0.0.1:53311/solr/.system_shard1_replica_n1/ returned schema version 0 and has not applied schema version 2
[junit4] 2> 406857 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.h.SchemaHandler REFRESHING SCHEMA (refreshIfBelowVersion=2, currentVersion=0) before returning version!
[junit4] 2> 406858 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 406858 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.S.Request [.system_shard1_replica_n1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=2&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 406858 ERROR (managedSchemaExecutor-1963-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Replica http://127.0.0.1:53311/solr/.system_shard1_replica_n1/ returned schema version 0 and has not applied schema version 2
[junit4] 2> 406859 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.h.SchemaHandler REFRESHING SCHEMA (refreshIfBelowVersion=2, currentVersion=0) before returning version!
[junit4] 2> 406860 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 406860 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.S.Request [.system_shard1_replica_n1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=2&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 406860 ERROR (managedSchemaExecutor-1963-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Replica http://127.0.0.1:53311/solr/.system_shard1_replica_n1/ returned schema version 0 and has not applied schema version 2
[junit4] 2> 406861 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.h.SchemaHandler REFRESHING SCHEMA (refreshIfBelowVersion=2, currentVersion=0) before returning version!
[junit4] 2> 406861 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 406861 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.S.Request [.system_shard1_replica_n1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=2&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 406861 ERROR (managedSchemaExecutor-1963-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Replica http://127.0.0.1:53311/solr/.system_shard1_replica_n1/ returned schema version 0 and has not applied schema version 2
[junit4] 2> 406862 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.h.SchemaHandler REFRESHING SCHEMA (refreshIfBelowVersion=2, currentVersion=0) before returning version!
[junit4] 2> 406863 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 406863 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.S.Request [.system_shard1_replica_n1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=2&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 406863 ERROR (managedSchemaExecutor-1963-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Replica http://127.0.0.1:53311/solr/.system_shard1_replica_n1/ returned schema version 0 and has not applied schema version 2
[junit4] 2> 406864 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.h.SchemaHandler REFRESHING SCHEMA (refreshIfBelowVersion=2, currentVersion=0) before returning version!
[junit4] 2> 406864 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 406864 INFO (qtp1650901900-6992) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.S.Request [.system_shard1_replica_n1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=2&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 406865 ERROR (managedSchemaExecutor-1963-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Replica http://127.0.0.1:53311/solr/.system_shard1_replica_n1/ returned schema version 0 and has not applied schema version 2
[junit4] 2> 406865 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.h.SchemaHandler REFRESHING SCHEMA (refreshIfBelowVersion=2, currentVersion=0) before returning version!
[junit4] 2> 406866 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 406866 INFO (qtp1650901900-6991) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.S.Request [.system_shard1_replica_n1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=2&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 406867 ERROR (managedSchemaExecutor-1963-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Replica http://127.0.0.1:53311/solr/.system_shard1_replica_n1/ returned schema version 0 and has not applied schema version 2
[junit4] 2> 406867 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.h.SchemaHandler REFRESHING SCHEMA (refreshIfBelowVersion=2, currentVersion=0) before returning version!
[junit4] 2> 406868 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 406868 INFO (qtp1650901900-6993) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1 ] o.a.s.c.S.Request [.system_shard1_replica_n1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=2&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 406868 ERROR (managedSchemaExecutor-1963-thread-1-processing-n:127.0.0.1:53311_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:53311_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Replica http://127.0.0.1:53311/solr/.system_shard1_replica_n1/ returned schema version 0 and has
[...truncated too long message...]
ionException: org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://127.0.0.1:63453/solr/.system_shard1_replica_n1
[junit4] 2> 155963 WARN (managedSchemaExecutor-37-thread-1-processing-n:127.0.0.1:63453_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:63453_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Failed to get /schema/zkversion from http://127.0.0.1:63453/solr/.system_shard1_replica_n1/ due to: java.util.concurrent.ExecutionException: org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://127.0.0.1:63453/solr/.system_shard1_replica_n1
[junit4] 2> 155964 WARN (managedSchemaExecutor-37-thread-1-processing-n:127.0.0.1:63453_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:63453_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Failed to get /schema/zkversion from http://127.0.0.1:63453/solr/.system_shard1_replica_n1/ due to: java.util.concurrent.ExecutionException: org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://127.0.0.1:63453/solr/.system_shard1_replica_n1
[junit4] 2> 155964 WARN (managedSchemaExecutor-37-thread-1-processing-n:127.0.0.1:63453_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:63453_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Failed to get /schema/zkversion from http://127.0.0.1:63453/solr/.system_shard1_replica_n1/ due to: java.util.concurrent.ExecutionException: org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://127.0.0.1:63453/solr/.system_shard1_replica_n1
[junit4] 2> 155965 WARN (managedSchemaExecutor-37-thread-1-processing-n:127.0.0.1:63453_solr x:.system_shard1_replica_n2 c:.system s:shard1 r:core_node4) [n:127.0.0.1:63453_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Failed to get /schema/zkversion from http://127.0.0.1:63453/solr/.system_shard1_replica_n1/ due to: java.util.concurrent.ExecutionException: org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://127.0.0.1:63453/solr/.system_shard1_replica_n1
[junit4] 2> 157299 INFO (httpUriRequest-33831-thread-1-processing-x:.system_shard1_replica_n2 r:core_node4 n:127.0.0.1:63453_solr http:////127.0.0.1:63453//solr//.system_shard1_replica_n1 c:.system s:shard1) [n:127.0.0.1:63453_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.h.i.e.RetryExec I/O exception (java.net.SocketException) caught when processing request to {}->http://127.0.0.1:63453: Socket closed
[junit4] 2> 157299 INFO (httpUriRequest-33831-thread-1-processing-x:.system_shard1_replica_n2 r:core_node4 n:127.0.0.1:63453_solr http:////127.0.0.1:63453//solr//.system_shard1_replica_n1 c:.system s:shard1) [n:127.0.0.1:63453_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.h.i.e.RetryExec Retrying request to {}->http://127.0.0.1:63453
[junit4] 2> 157301 WARN (qtp1719256256-55) [n:127.0.0.1:63453_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Core core_node4 was interrupted waiting for schema version 2 to propagate to 1 replicas for collection .system
[junit4] 2> 157301 INFO (qtp1719256256-55) [n:127.0.0.1:63453_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.s.ManagedIndexSchema Took 135068.0ms for 1 replicas to apply schema update version 2 for collection .system
[junit4] 2> 157301 INFO (qtp1719256256-55) [n:127.0.0.1:63453_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.S.Request [.system_shard1_replica_n2] webapp=/solr path=/schema params={wt=javabin&version=2} status=0 QTime=136064
[junit4] 2> 157301 INFO (qtp1719256256-55) [n:127.0.0.1:63453_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.c.SolrCore [.system_shard1_replica_n2] CLOSING SolrCore org.apache.solr.core.SolrCore@6a5f2420
[junit4] 2> 157301 WARN (qtp1719256256-55) [n:127.0.0.1:63453_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.m.SolrMetricManager Interrupted while trying to obtain lock to modify reporters registry: solr.core..system.shard1.replica_n2
[junit4] 2> 157301 INFO (qtp1719256256-55) [n:127.0.0.1:63453_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection..system.shard1.leader, tag=SolrCore@6a5f2420
[junit4] 2> 157302 INFO (qtp1719256256-55) [n:127.0.0.1:63453_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter.close() .
[junit4] 2> 157302 INFO (qtp1719256256-55) [n:127.0.0.1:63453_solr c:.system s:shard1 r:core_node4 x:.system_shard1_replica_n2 ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@7883863d commitCommandVersion:0
[junit4] 2> 157312 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
[junit4] 2> 157531 WARN (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/clusterprops.json
[junit4] 2> 7 /solr/aliases.json
[junit4] 2> 7 /solr/collections/.system/terms/shard1
[junit4] 2> 4 /solr/configs/.system/managed-schema
[junit4] 2> 2 /solr/configs/.system
[junit4] 2> 2 /solr/packages.json
[junit4] 2> 2 /solr/security.json
[junit4] 2> 2 /solr/collections/.system/collectionprops.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 15 /solr/collections/.system/state.json
[junit4] 2> 7 /solr/clusterstate.json
[junit4] 2> 6 /solr/aliases.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 25 /solr/overseer/queue
[junit4] 2> 18 /solr/collections
[junit4] 2> 11 /solr/overseer/collection-queue-work
[junit4] 2> 10 /solr/live_nodes
[junit4] 2>
[junit4] 2> 157537 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:63423
[junit4] 2> 157537 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:63423
[junit4] 2> 157537 INFO (SUITE-SystemCollectionCompatTest-seed#[B830D02E9D3F258C]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 63423
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J4/temp/solr.cloud.SystemCollectionCompatTest_B830D02E9D3F258C-001
[junit4] 2> Jan 27, 2020 2:06:48 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene84), sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@6833c8df), locale=hi, timezone=Africa/Malabo
[junit4] 2> NOTE: Mac OS X 10.14.6 x86_64/AdoptOpenJDK 12.0.2 (64-bit)/cpus=6,threads=2,free=89415400,total=194969600
[junit4] 2> NOTE: All tests run in this JVM: [SystemCollectionCompatTest]
[junit4] Completed [9/10 (1!)] on J4 in 156.56s, 1 test, 1 error <<< FAILURES!
[...truncated 25 lines...]
BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/lucene/common-build.xml:1510: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/lucene/common-build.xml:1037: There were test failures: 10 suites, 50 tests, 1 error [seed: B830D02E9D3F258C]
Total time: 3 minutes 26 seconds
[repro] Setting last failure code to 256
[repro] Failures w/original seeds:
[repro] 0/5 failed: org.apache.solr.cloud.DeleteReplicaTest
[repro] 1/5 failed: org.apache.solr.cloud.SystemCollectionCompatTest
[repro] Exiting with code 256
+ mv lucene/build lucene/build.repro
+ mv solr/build solr/build.repro
+ mv lucene/build.orig lucene/build
+ mv solr/build.orig solr/build
Archiving artifacts
Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
[WARNINGS] Parsing warnings in console log with parser Java Compiler (javac)
Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
No credentials specified
Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
<Git Blamer> Using GitBlamer to create author and commit information for all warnings.
<Git Blamer> GIT_COMMIT=6f85ec04602aa083b4512667d37e36e0213b5c35, workspace=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX
[WARNINGS] Computing warning deltas based on reference build #5574
Recording test results
Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk-13.0.2) - Build #
5576 - Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/5576/
Java: 64bit/jdk-13.0.2 -XX:-UseCompressedOops -XX:+UseSerialGC
5 tests failed.
FAILED: org.apache.solr.cloud.TestCloudSearcherWarming.testPeersyncFailureReplicationSuccess
Error Message:
KeeperErrorCode = Session expired for /clusterstate.json
Stack Trace:
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
at org.apache.zookeeper.KeeperException.create(KeeperException.java:134)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:2021)
at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:314)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:71)
at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:314)
at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:504)
at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:546)
at org.apache.solr.cloud.TestCloudSearcherWarming.tearDown(TestCloudSearcherWarming.java:79)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:567)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:1000)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
at java.base/java.lang.Thread.run(Thread.java:830)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestCloudSearcherWarming
Error Message:
47 threads leaked from SUITE scope at org.apache.solr.cloud.TestCloudSearcherWarming: 1) Thread[id=18141, name=qtp782580736-18141, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 2) Thread[id=17920, name=NIOWorkerThread-3, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 3) Thread[id=17919, name=NIOWorkerThread-2, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 4) Thread[id=17910, name=ConnnectionExpirer, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/java.lang.Thread.sleep(Native Method) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$ConnectionExpirerThread.run(NIOServerCnxnFactory.java:574) 5) Thread[id=18161, name=Thread-2459, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/java.lang.Object.wait(Native Method) at java.base@13.0.2/java.lang.Object.wait(Object.java:326) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2062) 6) Thread[id=17913, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/java.lang.Thread.sleep(Native Method) at app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:151) 7) Thread[id=18159, name=MetricsHistoryHandler-6501-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 8) Thread[id=18013, name=qtp1539586400-18013, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 9) Thread[id=18142, name=qtp782580736-18142, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 10) Thread[id=18143, name=Session-HouseKeeper-301de58f-1, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 11) Thread[id=17924, name=NIOWorkerThread-6, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 12) Thread[id=17922, name=NIOWorkerThread-4, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 13) Thread[id=18139, name=qtp782580736-18139, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 14) Thread[id=17934, name=NIOWorkerThread-11, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 15) Thread[id=17933, name=NIOWorkerThread-10, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 16) Thread[id=18007, name=qtp1539586400-18007, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/sun.nio.ch.KQueue.poll(Native Method) at java.base@13.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at java.base@13.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@13.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$189/0x000000014bd14458.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 17) Thread[id=18015, name=Session-HouseKeeper-1bb3e43d-1, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 18) Thread[id=18039, name=Connector-Scheduler-5fc87a1f-1, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 19) Thread[id=18012, name=qtp1539586400-18012, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 20) Thread[id=17912, name=NIOServerCxnFactory.AcceptThread:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/sun.nio.ch.KQueue.poll(Native Method) at java.base@13.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at java.base@13.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@13.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:229) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:205) 21) Thread[id=17925, name=NIOWorkerThread-7, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 22) Thread[id=18010, name=qtp1539586400-18010-acceptor-0@67695ce8-ServerConnector@5fc87a1f{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:61691}, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/sun.nio.ch.Net.accept(Native Method) at java.base@13.0.2/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:276) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:701) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 23) Thread[id=18014, name=qtp1539586400-18014, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 24) Thread[id=17936, name=NIOWorkerThread-12, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 25) Thread[id=17909, name=ZkTestServer Run Thread, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/java.lang.Object.wait(Native Method) at java.base@13.0.2/java.lang.Thread.join(Thread.java:1303) at java.base@13.0.2/java.lang.Thread.join(Thread.java:1371) at app//org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:950) at app//org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:343) at app//org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:566) 26) Thread[id=18137, name=qtp782580736-18137, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/sun.nio.ch.KQueue.poll(Native Method) at java.base@13.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at java.base@13.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@13.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$189/0x000000014bd14458.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 27) Thread[id=18011, name=qtp1539586400-18011, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 28) Thread[id=18030, name=SolrRrdBackendFactory-6463-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 29) Thread[id=17928, name=NIOWorkerThread-8, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 30) Thread[id=17929, name=NIOWorkerThread-9, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 31) Thread[id=18173, name=Connector-Scheduler-7f9df35d-1, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 32) Thread[id=18009, name=qtp1539586400-18009, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/sun.nio.ch.KQueue.poll(Native Method) at java.base@13.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at java.base@13.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@13.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$189/0x000000014bd14458.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 33) Thread[id=17923, name=NIOWorkerThread-5, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 34) Thread[id=18031, name=MetricsHistoryHandler-6464-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 35) Thread[id=18032, name=Thread-2441, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/java.lang.Object.wait(Native Method) at java.base@13.0.2/java.lang.Object.wait(Object.java:326) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2062) 36) Thread[id=18136, name=qtp782580736-18136, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/sun.nio.ch.KQueue.poll(Native Method) at java.base@13.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at java.base@13.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@13.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$189/0x000000014bd14458.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 37) Thread[id=18172, name=SolrRrdBackendFactory-6463-thread-2, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 38) Thread[id=18008, name=qtp1539586400-18008, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/sun.nio.ch.KQueue.poll(Native Method) at java.base@13.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at java.base@13.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@13.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$189/0x000000014bd14458.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 39) Thread[id=18140, name=qtp782580736-18140, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 40) Thread[id=18135, name=qtp782580736-18135, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/sun.nio.ch.KQueue.poll(Native Method) at java.base@13.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at java.base@13.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@13.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$189/0x000000014bd14458.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 41) Thread[id=18138, name=qtp782580736-18138-acceptor-0@78c6185f-ServerConnector@7f9df35d{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:61896}, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/sun.nio.ch.Net.accept(Native Method) at java.base@13.0.2/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:276) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:701) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 42) Thread[id=18158, name=SolrRrdBackendFactory-6500-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 43) Thread[id=17914, name=SyncThread:0, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at app//org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:109) 44) Thread[id=17911, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/sun.nio.ch.KQueue.poll(Native Method) at java.base@13.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122) at java.base@13.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@13.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:426) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:391) 45) Thread[id=17915, name=ProcessThread(sid:0 cport:61604):, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at app//org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:134) 46) Thread[id=17916, name=NIOWorkerThread-1, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830) 47) Thread[id=18180, name=SolrRrdBackendFactory-6500-thread-2, state=WAITING, group=TGRP-TestCloudSearcherWarming] at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method) at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 47 threads leaked from SUITE scope at org.apache.solr.cloud.TestCloudSearcherWarming:
1) Thread[id=18141, name=qtp782580736-18141, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
2) Thread[id=17920, name=NIOWorkerThread-3, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
3) Thread[id=17919, name=NIOWorkerThread-2, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
4) Thread[id=17910, name=ConnnectionExpirer, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/java.lang.Thread.sleep(Native Method)
at app//org.apache.zookeeper.server.NIOServerCnxnFactory$ConnectionExpirerThread.run(NIOServerCnxnFactory.java:574)
5) Thread[id=18161, name=Thread-2459, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/java.lang.Object.wait(Native Method)
at java.base@13.0.2/java.lang.Object.wait(Object.java:326)
at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2062)
6) Thread[id=17913, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/java.lang.Thread.sleep(Native Method)
at app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:151)
7) Thread[id=18159, name=MetricsHistoryHandler-6501-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
8) Thread[id=18013, name=qtp1539586400-18013, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
9) Thread[id=18142, name=qtp782580736-18142, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
10) Thread[id=18143, name=Session-HouseKeeper-301de58f-1, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
11) Thread[id=17924, name=NIOWorkerThread-6, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
12) Thread[id=17922, name=NIOWorkerThread-4, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
13) Thread[id=18139, name=qtp782580736-18139, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
14) Thread[id=17934, name=NIOWorkerThread-11, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
15) Thread[id=17933, name=NIOWorkerThread-10, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
16) Thread[id=18007, name=qtp1539586400-18007, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/sun.nio.ch.KQueue.poll(Native Method)
at java.base@13.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
at java.base@13.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
at java.base@13.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$189/0x000000014bd14458.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
17) Thread[id=18015, name=Session-HouseKeeper-1bb3e43d-1, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
18) Thread[id=18039, name=Connector-Scheduler-5fc87a1f-1, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
19) Thread[id=18012, name=qtp1539586400-18012, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
20) Thread[id=17912, name=NIOServerCxnFactory.AcceptThread:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/sun.nio.ch.KQueue.poll(Native Method)
at java.base@13.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
at java.base@13.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
at java.base@13.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
at app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:229)
at app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:205)
21) Thread[id=17925, name=NIOWorkerThread-7, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
22) Thread[id=18010, name=qtp1539586400-18010-acceptor-0@67695ce8-ServerConnector@5fc87a1f{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:61691}, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/sun.nio.ch.Net.accept(Native Method)
at java.base@13.0.2/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:276)
at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:701)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
23) Thread[id=18014, name=qtp1539586400-18014, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
24) Thread[id=17936, name=NIOWorkerThread-12, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
25) Thread[id=17909, name=ZkTestServer Run Thread, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/java.lang.Object.wait(Native Method)
at java.base@13.0.2/java.lang.Thread.join(Thread.java:1303)
at java.base@13.0.2/java.lang.Thread.join(Thread.java:1371)
at app//org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:950)
at app//org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:343)
at app//org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:566)
26) Thread[id=18137, name=qtp782580736-18137, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/sun.nio.ch.KQueue.poll(Native Method)
at java.base@13.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
at java.base@13.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
at java.base@13.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$189/0x000000014bd14458.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
27) Thread[id=18011, name=qtp1539586400-18011, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
28) Thread[id=18030, name=SolrRrdBackendFactory-6463-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
29) Thread[id=17928, name=NIOWorkerThread-8, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
30) Thread[id=17929, name=NIOWorkerThread-9, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
31) Thread[id=18173, name=Connector-Scheduler-7f9df35d-1, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
32) Thread[id=18009, name=qtp1539586400-18009, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/sun.nio.ch.KQueue.poll(Native Method)
at java.base@13.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
at java.base@13.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
at java.base@13.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$189/0x000000014bd14458.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
33) Thread[id=17923, name=NIOWorkerThread-5, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
34) Thread[id=18031, name=MetricsHistoryHandler-6464-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
35) Thread[id=18032, name=Thread-2441, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/java.lang.Object.wait(Native Method)
at java.base@13.0.2/java.lang.Object.wait(Object.java:326)
at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2062)
36) Thread[id=18136, name=qtp782580736-18136, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/sun.nio.ch.KQueue.poll(Native Method)
at java.base@13.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
at java.base@13.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
at java.base@13.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$189/0x000000014bd14458.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
37) Thread[id=18172, name=SolrRrdBackendFactory-6463-thread-2, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
38) Thread[id=18008, name=qtp1539586400-18008, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/sun.nio.ch.KQueue.poll(Native Method)
at java.base@13.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
at java.base@13.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
at java.base@13.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$189/0x000000014bd14458.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
39) Thread[id=18140, name=qtp782580736-18140, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:875)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:925)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
40) Thread[id=18135, name=qtp782580736-18135, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/sun.nio.ch.KQueue.poll(Native Method)
at java.base@13.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
at java.base@13.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
at java.base@13.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:472)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:409)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$189/0x000000014bd14458.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
41) Thread[id=18138, name=qtp782580736-18138-acceptor-0@78c6185f-ServerConnector@7f9df35d{ssl,[ssl, alpn, http/1.1, h2]}{127.0.0.1:61896}, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/sun.nio.ch.Net.accept(Native Method)
at java.base@13.0.2/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:276)
at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385)
at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:701)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
42) Thread[id=18158, name=SolrRrdBackendFactory-6500-thread-1, state=TIMED_WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@13.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
43) Thread[id=17914, name=SyncThread:0, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
at app//org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:109)
44) Thread[id=17911, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/sun.nio.ch.KQueue.poll(Native Method)
at java.base@13.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
at java.base@13.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
at java.base@13.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:426)
at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:391)
45) Thread[id=17915, name=ProcessThread(sid:0 cport:61604):, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
at app//org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:134)
46) Thread[id=17916, name=NIOWorkerThread-1, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
at java.base@13.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
at java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
47) Thread[id=18180, name=SolrRrdBackendFactory-6500-thread-2, state=WAITING, group=TGRP-TestCloudSearcherWarming]
at java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
at java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchr
[...truncated too long message...]
_policy/terms/s3
[junit4] 2> 3 /solr/collections/testCreateCollection/terms/shard2
[junit4] 2> 3 /solr/collections/addshard_with_reptype_using_policy/state.json
[junit4] 2> 3 /solr/collections/addshard_using_policy/terms/s2
[junit4] 2> 3 /solr/collections/metrics_tags/terms/shard1
[junit4] 2> 2 /solr/collections/testCreateCollectionSplitShard/collectionprops.json
[junit4] 2> 2 /solr/collections/testCreateCollection/terms/shard1
[junit4] 2> 2 /solr/collections/addshard_using_policy/terms/s1
[junit4] 2> 2 /solr/collections/addshard_with_reptype_using_policy/collectionprops.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 124 /solr/collections/addshard_with_reptype_using_policy/state.json
[junit4] 2> 84 /solr/aliases.json
[junit4] 2> 81 /solr/collections/testCreateCollectionSplitShard/state.json
[junit4] 2> 43 /solr/collections/addshard_using_policy/state.json
[junit4] 2> 41 /solr/collections/perReplicaDataColl/state.json
[junit4] 2> 28 /solr/collections/testCreateCollectionAddReplica/state.json
[junit4] 2> 26 /solr/collections/testCreateCollection/state.json
[junit4] 2> 17 /solr/collections/data_provider/state.json
[junit4] 2> 16 /solr/autoscaling.json
[junit4] 2> 14 /solr/clusterprops.json
[junit4] 2> 14 /solr/clusterstate.json
[junit4] 2> 13 /solr/collections/metrics_tags/state.json
[junit4] 2> 3 /solr/overseer_elect/election/72076454885785616-127.0.0.1:61457_solr-n_0000000000
[junit4] 2> 3 /solr/collections/perReplicaDataColl/leader_elect/shard1/election/72076454885785616-core_node5-n_0000000001
[junit4] 2> 2 /solr/collections/perReplicaDataColl/leader_elect/shard1/election/72076454885785613-core_node10-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 237 /solr/overseer/queue
[junit4] 2> 130 /solr/collections
[junit4] 2> 75 /solr/overseer/collection-queue-work
[junit4] 2> 41 /solr/live_nodes
[junit4] 2>
[junit4] 2> 1748880 INFO (SUITE-TestPolicyCloud-seed#[4A92D2EB4FF4301A]-worker) [ ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:61449
[junit4] 2> 1748880 INFO (SUITE-TestPolicyCloud-seed#[4A92D2EB4FF4301A]-worker) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:61449
[junit4] 2> 1748880 INFO (SUITE-TestPolicyCloud-seed#[4A92D2EB4FF4301A]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 61449
[junit4] 2> 1748881 INFO (SUITE-TestPolicyCloud-seed#[4A92D2EB4FF4301A]-worker) [ ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
[junit4] 2> Jan 27, 2020 6:48:58 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> Jan 27, 2020 6:49:08 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> SEVERE: 1 thread leaked from SUITE scope at org.apache.solr.cloud.autoscaling.TestPolicyCloud:
[junit4] 2> 1) Thread[id=22904, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestPolicyCloud]
[junit4] 2> at java.base@13.0.2/java.lang.Thread.sleep(Native Method)
[junit4] 2> at app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:151)
[junit4] 2> Jan 27, 2020 6:49:08 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: Starting to interrupt leaked threads:
[junit4] 2> 1) Thread[id=22904, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestPolicyCloud]
[junit4] 2> Jan 27, 2020 6:49:08 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: All leaked threads terminated.
[junit4] 2> 1758947 ERROR (SessionTracker) [ ] o.a.z.s.ZooKeeperCriticalThread Severe unrecoverable error, from thread : SessionTracker
[junit4] 2> => java.lang.InterruptedException: sleep interrupted
[junit4] 2> at java.base/java.lang.Thread.sleep(Native Method)
[junit4] 2> java.lang.InterruptedException: sleep interrupted
[junit4] 2> at java.lang.Thread.sleep(Native Method) [?:?]
[junit4] 2> at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:151) ~[zookeeper-3.5.5.jar:3.5.5]
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene84): {}, docValues:{}, maxPointsInLeafNode=1628, maxMBSortInHeap=5.319995602412868, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@eaef1e8), locale=mfe-MU, timezone=Europe/Ulyanovsk
[junit4] 2> NOTE: Mac OS X 10.14.6 x86_64/AdoptOpenJDK 13.0.2 (64-bit)/cpus=6,threads=1,free=219864560,total=503713792
[junit4] 2> NOTE: All tests run in this JVM: [IndexSizeTriggerTest, XmlInterpolationTest, TestPerFieldSimilarityWithDefaultOverride, TestUpdate, TestInPlaceUpdatesDistrib, XCJFQueryTest, TestNumericTerms32, AnalysisAfterCoreReloadTest, PropertiesRequestHandlerTest, ScheduledTriggerIntegrationTest, TestOverriddenPrefixQueryForCustomFieldType, DateMathParserTest, TestDirectoryFactory, CursorMarkTest, TestCloudSchemaless, EmbeddedSolrNoSerializeTest, AlternateDirectoryTest, CollectionTooManyReplicasTest, DistributedQueueTest, IgnoreLargeDocumentProcessorFactoryTest, SuggesterTSTTest, ExecutePlanActionTest, PhrasesIdentificationComponentTest, DimensionalRoutedAliasUpdateProcessorTest, CollectionsAPISolrJTest, IndexSchemaRuntimeFieldTest, BasicAuthIntegrationTest, OverriddenZkACLAndCredentialsProvidersTest, TestTrieFacet, DistributedFacetPivotLargeTest, RequiredFieldsTest, TestSolrCloudWithSecureImpersonation, LoggingHandlerTest, TestSimClusterStateProvider, ChangedSchemaMergeTest, TestMacroExpander, AdminHandlersProxyTest, ZkShardTermsTest, TestCloudConsistency, QueryParsingTest, TestMinMaxOnMultiValuedField, UpdateRequestProcessorFactoryTest, TestSolrQueryParser, TestNRTOpen, TestMaxScoreQueryParser, TestManagedSchemaAPI, TestSchemaSimilarityResource, TestReplicationHandler, TestJoin, TestWithCollection, PluginInfoTest, AssignBackwardCompatibilityTest, TestSubQueryTransformerCrossCore, TestFieldResource, HdfsRecoveryZkTest, TestCodecSupport, TestBlobHandler, ConfigureRecoveryStrategyTest, TestManagedSynonymFilterFactory, TestInfoStreamLogging, FullSolrCloudDistribCmdsTest, SplitHandlerTest, CloudExitableDirectoryReaderTest, TestSimpleTextCodec, TestShortCircuitedRequests, TestExactSharedStatsCache, RangeFacetCloudTest, TestRequestStatusCollectionAPI, TestBM25SimilarityFactory, CoreAdminRequestStatusTest, TestJsonFacets, CheckHdfsIndexTest, TestPolicyCloud]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestPolicyCloud -Dtests.seed=4A92D2EB4FF4301A -Dtests.slow=true -Dtests.locale=mfe-MU -Dtests.timezone=Europe/Ulyanovsk -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 0.00s J0 | TestPolicyCloud (suite) <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.autoscaling.TestPolicyCloud:
[junit4] > 1) Thread[id=22904, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestPolicyCloud]
[junit4] > at java.base@13.0.2/java.lang.Thread.sleep(Native Method)
[junit4] > at app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:151)
[junit4] > at __randomizedtesting.SeedInfo.seed([4A92D2EB4FF4301A]:0)
[junit4] Completed [561/900 (3!)] on J0 in 284.57s, 8 tests, 1 error <<< FAILURES!
[...truncated 45814 lines...]
[repro] Jenkins log URL: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/5576/consoleText
[repro] Revision: 6f85ec04602aa083b4512667d37e36e0213b5c35
[repro] Ant options: "-Dargs=-XX:-UseCompressedOops -XX:+UseSerialGC"
[repro] Repro line: ant test -Dtestcase=TestConfigSetsAPI -Dtests.seed=4A92D2EB4FF4301A -Dtests.slow=true -Dtests.locale=nus -Dtests.timezone=America/Argentina/San_Luis -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[repro] Repro line: ant test -Dtestcase=TestCloudSearcherWarming -Dtests.method=testPeersyncFailureReplicationSuccess -Dtests.seed=4A92D2EB4FF4301A -Dtests.slow=true -Dtests.locale=zh-Hans-MO -Dtests.timezone=America/Guyana -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[repro] Repro line: ant test -Dtestcase=TestCloudSearcherWarming -Dtests.seed=4A92D2EB4FF4301A -Dtests.slow=true -Dtests.locale=zh-Hans-MO -Dtests.timezone=America/Guyana -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[repro] Repro line: ant test -Dtestcase=TestPolicyCloud -Dtests.seed=4A92D2EB4FF4301A -Dtests.slow=true -Dtests.locale=mfe-MU -Dtests.timezone=Europe/Ulyanovsk -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[repro] JUnit rest result XML files will be moved to: ./repro-reports
[repro] ant clean
[...truncated 6 lines...]
[repro] Test suites by module:
[repro] solr/core
[repro] TestConfigSetsAPI
[repro] TestCloudSearcherWarming
[repro] TestPolicyCloud
[repro] ant compile-test
[...truncated 3408 lines...]
[repro] ant test-nocompile -Dtests.dups=5 -Dtests.maxfailures=15 -Dtests.class="*.TestConfigSetsAPI|*.TestCloudSearcherWarming|*.TestPolicyCloud" -Dtests.showOutput=onerror "-Dargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Dtests.seed=4A92D2EB4FF4301A -Dtests.slow=true -Dtests.locale=nus -Dtests.timezone=America/Argentina/San_Luis -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[...truncated 206 lines...]
[repro] Failures w/original seeds:
[repro] 0/5 failed: org.apache.solr.cloud.TestCloudSearcherWarming
[repro] 0/5 failed: org.apache.solr.cloud.TestConfigSetsAPI
[repro] 0/5 failed: org.apache.solr.cloud.autoscaling.TestPolicyCloud
[repro] Exiting with code 0
[...truncated 73 lines...]