You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2018/09/01 08:22:12 UTC
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 792 -
Still Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/792/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC
5 tests failed.
FAILED: org.apache.solr.handler.RequestLoggingTest.testLogBeforeExecuteWithRequestLogger
Error Message:
Stack Trace:
java.lang.AssertionError
at __randomizedtesting.SeedInfo.seed([720E9648BD5B3057:C367B64710AD8819]:0)
at org.junit.Assert.fail(Assert.java:92)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertTrue(Assert.java:54)
at org.apache.solr.handler.RequestLoggingTest.testLogBeforeExecute(RequestLoggingTest.java:88)
at org.apache.solr.handler.RequestLoggingTest.testLogBeforeExecuteWithRequestLogger(RequestLoggingTest.java:71)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.handler.RequestLoggingTest.testLogBeforeExecuteWithRequestLogger
Error Message:
Stack Trace:
java.lang.AssertionError
at __randomizedtesting.SeedInfo.seed([720E9648BD5B3057:C367B64710AD8819]:0)
at org.junit.Assert.fail(Assert.java:92)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertTrue(Assert.java:54)
at org.apache.solr.handler.RequestLoggingTest.testLogBeforeExecute(RequestLoggingTest.java:88)
at org.apache.solr.handler.RequestLoggingTest.testLogBeforeExecuteWithRequestLogger(RequestLoggingTest.java:71)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.handler.RequestLoggingTest.testLogBeforeExecuteWithCoreLogger
Error Message:
Stack Trace:
java.lang.AssertionError
at __randomizedtesting.SeedInfo.seed([720E9648BD5B3057:BE8896C643925BC1]:0)
at org.junit.Assert.fail(Assert.java:92)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertTrue(Assert.java:54)
at org.apache.solr.handler.RequestLoggingTest.testLogBeforeExecute(RequestLoggingTest.java:88)
at org.apache.solr.handler.RequestLoggingTest.testLogBeforeExecuteWithCoreLogger(RequestLoggingTest.java:65)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.cdcr.CdcrBidirectionalTest.testBiDir
Error Message:
Captured an uncaught exception in thread: Thread[id=717, name=cdcr-replicator-201-thread-1, state=RUNNABLE, group=TGRP-CdcrBidirectionalTest]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=717, name=cdcr-replicator-201-thread-1, state=RUNNABLE, group=TGRP-CdcrBidirectionalTest]
Caused by: java.lang.AssertionError: 1610392463645605888 != 1610392463644557312
at __randomizedtesting.SeedInfo.seed([720E9648BD5B3057]:0)
at org.apache.solr.update.CdcrUpdateLog$CdcrLogReader.forwardSeek(CdcrUpdateLog.java:611)
at org.apache.solr.handler.CdcrReplicator.run(CdcrReplicator.java:105)
at org.apache.solr.handler.CdcrReplicatorScheduler.lambda$null$0(CdcrReplicatorScheduler.java:81)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.cdcr.CdcrBidirectionalTest.testBiDir
Error Message:
Captured an uncaught exception in thread: Thread[id=5516, name=cdcr-replicator-1516-thread-1, state=RUNNABLE, group=TGRP-CdcrBidirectionalTest]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=5516, name=cdcr-replicator-1516-thread-1, state=RUNNABLE, group=TGRP-CdcrBidirectionalTest]
Caused by: java.lang.AssertionError: 1610386842578845696 != 1610386842577797120
at __randomizedtesting.SeedInfo.seed([720E9648BD5B3057]:0)
at org.apache.solr.update.CdcrUpdateLog$CdcrLogReader.forwardSeek(CdcrUpdateLog.java:611)
at org.apache.solr.handler.CdcrReplicator.run(CdcrReplicator.java:125)
at org.apache.solr.handler.CdcrReplicatorScheduler.lambda$null$0(CdcrReplicatorScheduler.java:81)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12580 lines...]
[junit4] Suite: org.apache.solr.cloud.cdcr.CdcrBidirectionalTest
[junit4] 2> 296139 INFO (SUITE-CdcrBidirectionalTest-seed#[720E9648BD5B3057]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBidirectionalTest_720E9648BD5B3057-001/init-core-data-001
[junit4] 2> 296139 WARN (SUITE-CdcrBidirectionalTest-seed#[720E9648BD5B3057]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
[junit4] 2> 296139 INFO (SUITE-CdcrBidirectionalTest-seed#[720E9648BD5B3057]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 296140 INFO (SUITE-CdcrBidirectionalTest-seed#[720E9648BD5B3057]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 296142 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[720E9648BD5B3057]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testBiDir
[junit4] 2> 296142 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[720E9648BD5B3057]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBidirectionalTest_720E9648BD5B3057-001/cdcr-cluster2-001
[junit4] 2> 296142 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[720E9648BD5B3057]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 296143 INFO (Thread-673) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 296143 INFO (Thread-673) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 296144 ERROR (Thread-673) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 296243 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[720E9648BD5B3057]) [ ] o.a.s.c.ZkTestServer start zk server on port:50280
[junit4] 2> 296245 INFO (zkConnectionManagerCallback-2128-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 296254 INFO (jetty-launcher-2125-thread-1) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 296255 INFO (jetty-launcher-2125-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 296255 INFO (jetty-launcher-2125-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 296255 INFO (jetty-launcher-2125-thread-1) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 296256 INFO (jetty-launcher-2125-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5272e904{/solr,null,AVAILABLE}
[junit4] 2> 296258 INFO (jetty-launcher-2125-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@66486e98{SSL,[ssl, http/1.1]}{127.0.0.1:64026}
[junit4] 2> 296258 INFO (jetty-launcher-2125-thread-1) [ ] o.e.j.s.Server Started @296432ms
[junit4] 2> 296258 INFO (jetty-launcher-2125-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=64026}
[junit4] 2> 296258 ERROR (jetty-launcher-2125-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 296258 INFO (jetty-launcher-2125-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 296258 INFO (jetty-launcher-2125-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.5.0
[junit4] 2> 296258 INFO (jetty-launcher-2125-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 296258 INFO (jetty-launcher-2125-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 296258 INFO (jetty-launcher-2125-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-09-01T06:49:07.795Z
[junit4] 2> 296260 INFO (zkConnectionManagerCallback-2130-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 296261 INFO (jetty-launcher-2125-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 296755 INFO (jetty-launcher-2125-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50280/solr
[junit4] 2> 296757 INFO (zkConnectionManagerCallback-2134-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 296760 INFO (zkConnectionManagerCallback-2136-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 296890 INFO (jetty-launcher-2125-thread-1) [n:127.0.0.1:64026_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:64026_solr
[junit4] 2> 296891 INFO (jetty-launcher-2125-thread-1) [n:127.0.0.1:64026_solr ] o.a.s.c.Overseer Overseer (id=72259898766131203-127.0.0.1:64026_solr-n_0000000000) starting
[junit4] 2> 296901 INFO (zkConnectionManagerCallback-2143-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 296904 INFO (jetty-launcher-2125-thread-1) [n:127.0.0.1:64026_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50280/solr ready
[junit4] 2> 296905 INFO (jetty-launcher-2125-thread-1) [n:127.0.0.1:64026_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64026_solr
[junit4] 2> 296906 INFO (OverseerStateUpdate-72259898766131203-127.0.0.1:64026_solr-n_0000000000) [n:127.0.0.1:64026_solr ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:64026_solr
[junit4] 2> 296908 INFO (OverseerStateUpdate-72259898766131203-127.0.0.1:64026_solr-n_0000000000) [n:127.0.0.1:64026_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 296909 INFO (zkCallback-2142-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 296930 INFO (jetty-launcher-2125-thread-1) [n:127.0.0.1:64026_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 296950 INFO (jetty-launcher-2125-thread-1) [n:127.0.0.1:64026_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_64026.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@17547a5f
[junit4] 2> 296955 INFO (jetty-launcher-2125-thread-1) [n:127.0.0.1:64026_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_64026.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@17547a5f
[junit4] 2> 296955 INFO (jetty-launcher-2125-thread-1) [n:127.0.0.1:64026_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_64026.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@17547a5f
[junit4] 2> 296956 INFO (jetty-launcher-2125-thread-1) [n:127.0.0.1:64026_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBidirectionalTest_720E9648BD5B3057-001/cdcr-cluster2-001/node1/.
[junit4] 2> 296971 INFO (zkConnectionManagerCallback-2146-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 296972 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x100b7febd760005, likely client has closed socket
[junit4] 2> 296975 INFO (zkConnectionManagerCallback-2149-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 296977 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[720E9648BD5B3057]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBidirectionalTest_720E9648BD5B3057-001/cdcr-cluster1-001
[junit4] 2> 296977 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[720E9648BD5B3057]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 296978 INFO (Thread-683) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 296978 INFO (Thread-683) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 296980 ERROR (Thread-683) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 297077 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[720E9648BD5B3057]) [ ] o.a.s.c.ZkTestServer start zk server on port:46392
[junit4] 2> 297080 INFO (zkConnectionManagerCallback-2153-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 297087 INFO (jetty-launcher-2150-thread-1) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 297088 INFO (jetty-launcher-2150-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 297088 INFO (jetty-launcher-2150-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 297088 INFO (jetty-launcher-2150-thread-1) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 297089 INFO (jetty-launcher-2150-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@20dea252{/solr,null,AVAILABLE}
[junit4] 2> 297089 INFO (jetty-launcher-2150-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@ceef865{SSL,[ssl, http/1.1]}{127.0.0.1:59291}
[junit4] 2> 297090 INFO (jetty-launcher-2150-thread-1) [ ] o.e.j.s.Server Started @297264ms
[junit4] 2> 297090 INFO (jetty-launcher-2150-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=59291}
[junit4] 2> 297090 ERROR (jetty-launcher-2150-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 297090 INFO (jetty-launcher-2150-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 297090 INFO (jetty-launcher-2150-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.5.0
[junit4] 2> 297090 INFO (jetty-launcher-2150-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 297090 INFO (jetty-launcher-2150-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 297090 INFO (jetty-launcher-2150-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-09-01T06:49:08.627Z
[junit4] 2> 297091 INFO (zkConnectionManagerCallback-2155-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 297092 INFO (jetty-launcher-2150-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 298533 INFO (jetty-launcher-2150-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46392/solr
[junit4] 2> 298534 INFO (zkConnectionManagerCallback-2159-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 298537 INFO (zkConnectionManagerCallback-2161-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 298652 INFO (jetty-launcher-2150-thread-1) [n:127.0.0.1:59291_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59291_solr
[junit4] 2> 298654 INFO (jetty-launcher-2150-thread-1) [n:127.0.0.1:59291_solr ] o.a.s.c.Overseer Overseer (id=72259898820919299-127.0.0.1:59291_solr-n_0000000000) starting
[junit4] 2> 298664 INFO (zkConnectionManagerCallback-2168-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 298668 INFO (jetty-launcher-2150-thread-1) [n:127.0.0.1:59291_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:46392/solr ready
[junit4] 2> 298669 INFO (OverseerStateUpdate-72259898820919299-127.0.0.1:59291_solr-n_0000000000) [n:127.0.0.1:59291_solr ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:59291_solr
[junit4] 2> 298670 INFO (jetty-launcher-2150-thread-1) [n:127.0.0.1:59291_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59291_solr
[junit4] 2> 298674 INFO (OverseerStateUpdate-72259898820919299-127.0.0.1:59291_solr-n_0000000000) [n:127.0.0.1:59291_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 298684 INFO (zkCallback-2167-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 298696 INFO (jetty-launcher-2150-thread-1) [n:127.0.0.1:59291_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 298710 INFO (jetty-launcher-2150-thread-1) [n:127.0.0.1:59291_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59291.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@17547a5f
[junit4] 2> 298715 INFO (jetty-launcher-2150-thread-1) [n:127.0.0.1:59291_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59291.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@17547a5f
[junit4] 2> 298715 INFO (jetty-launcher-2150-thread-1) [n:127.0.0.1:59291_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59291.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@17547a5f
[junit4] 2> 298716 INFO (jetty-launcher-2150-thread-1) [n:127.0.0.1:59291_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBidirectionalTest_720E9648BD5B3057-001/cdcr-cluster1-001/node1/.
[junit4] 2> 298733 INFO (zkConnectionManagerCallback-2171-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 298735 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x100b7fec0ba0005, likely client has closed socket
[junit4] 2> 298738 INFO (zkConnectionManagerCallback-2174-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 298739 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[720E9648BD5B3057]) [ ] o.a.s.c.c.CdcrBidirectionalTest cluster2 zkHost = 127.0.0.1:50280/solr
[junit4] 2> 298739 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[720E9648BD5B3057]) [ ] o.a.s.c.c.CdcrBidirectionalTest cluster1 zkHost = 127.0.0.1:46392/solr
[junit4] 2> 298741 INFO (zkConnectionManagerCallback-2176-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 298747 INFO (zkConnectionManagerCallback-2180-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 298748 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[720E9648BD5B3057]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 298750 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[720E9648BD5B3057]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:46392/solr ready
[junit4] 2> 298765 INFO (qtp280651314-5378) [n:127.0.0.1:59291_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=cdcr-cluster1&maxShardsPerNode=2&name=cdcr-cluster1&nrtReplicas=1&action=CREATE&numShards=2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 298768 INFO (OverseerThreadFactory-1477-thread-1-processing-n:127.0.0.1:59291_solr) [n:127.0.0.1:59291_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection cdcr-cluster1
[junit4] 2> 298883 INFO (OverseerStateUpdate-72259898820919299-127.0.0.1:59291_solr-n_0000000000) [n:127.0.0.1:59291_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"cdcr-cluster1",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"cdcr-cluster1_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:59291/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 298884 INFO (OverseerStateUpdate-72259898820919299-127.0.0.1:59291_solr-n_0000000000) [n:127.0.0.1:59291_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"cdcr-cluster1",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"cdcr-cluster1_shard2_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:59291/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 299094 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr x:cdcr-cluster1_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=cdcr-cluster1&newCollection=true&collection=cdcr-cluster1&version=2&replicaType=NRT&coreNodeName=core_node3&name=cdcr-cluster1_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin
[junit4] 2> 299094 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 299095 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=cdcr-cluster1&newCollection=true&collection=cdcr-cluster1&version=2&replicaType=NRT&coreNodeName=core_node4&name=cdcr-cluster1_shard2_replica_n2&action=CREATE&numShards=2&shard=shard2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin
[junit4] 2> 300109 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.5.0
[junit4] 2> 300122 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.5.0
[junit4] 2> 300134 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.s.IndexSchema [cdcr-cluster1_shard1_replica_n1] Schema name=minimal
[junit4] 2> 300134 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.s.IndexSchema [cdcr-cluster1_shard2_replica_n2] Schema name=minimal
[junit4] 2> 300137 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 300137 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 300137 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'cdcr-cluster1_shard2_replica_n2' using configuration from collection cdcr-cluster1, trusted=true
[junit4] 2> 300137 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'cdcr-cluster1_shard1_replica_n1' using configuration from collection cdcr-cluster1, trusted=true
[junit4] 2> 300137 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59291.solr.core.cdcr-cluster1.shard2.replica_n2' (registry 'solr.core.cdcr-cluster1.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@17547a5f
[junit4] 2> 300137 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59291.solr.core.cdcr-cluster1.shard1.replica_n1' (registry 'solr.core.cdcr-cluster1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@17547a5f
[junit4] 2> 300143 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 300143 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.SolrCore [[cdcr-cluster1_shard2_replica_n2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBidirectionalTest_720E9648BD5B3057-001/cdcr-cluster1-001/node1/cdcr-cluster1_shard2_replica_n2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBidirectionalTest_720E9648BD5B3057-001/cdcr-cluster1-001/node1/./cdcr-cluster1_shard2_replica_n2/data/]
[junit4] 2> 300143 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 300143 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.SolrCore [[cdcr-cluster1_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBidirectionalTest_720E9648BD5B3057-001/cdcr-cluster1-001/node1/cdcr-cluster1_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBidirectionalTest_720E9648BD5B3057-001/cdcr-cluster1-001/node1/./cdcr-cluster1_shard1_replica_n1/data/]
[junit4] 2> 300362 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
[junit4] 2> 300362 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
[junit4] 2> 300362 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 300362 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 300364 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 300364 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 300364 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 300364 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 300366 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@495c0691[cdcr-cluster1_shard1_replica_n1] main]
[junit4] 2> 300366 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@7342e637[cdcr-cluster1_shard2_replica_n2] main]
[junit4] 2> 300368 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/cdcr-cluster1
[junit4] 2> 300368 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/cdcr-cluster1
[junit4] 2> 300368 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/cdcr-cluster1
[junit4] 2> 300368 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/cdcr-cluster1
[junit4] 2> 300371 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/cdcr-cluster1/managed-schema
[junit4] 2> 300371 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/cdcr-cluster1/managed-schema
[junit4] 2> 300373 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 300373 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 300373 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 300374 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 300382 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.h.CdcrBufferStateManager Created znode /collections/cdcr-cluster1/cdcr/state/buffer
[junit4] 2> 300391 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.h.CdcrProcessStateManager Created znode /collections/cdcr-cluster1/cdcr/state/process
[junit4] 2> 300427 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1610386822360203264
[junit4] 2> 300428 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1610386822361251840
[junit4] 2> 300428 INFO (searcherExecutor-1483-thread-1-processing-n:127.0.0.1:59291_solr x:cdcr-cluster1_shard2_replica_n2 c:cdcr-cluster1 s:shard2 r:core_node4) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.SolrCore [cdcr-cluster1_shard2_replica_n2] Registered new searcher Searcher@7342e637[cdcr-cluster1_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 300430 INFO (searcherExecutor-1482-thread-1-processing-n:127.0.0.1:59291_solr x:cdcr-cluster1_shard1_replica_n1 c:cdcr-cluster1 s:shard1 r:core_node3) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-cluster1_shard1_replica_n1] Registered new searcher Searcher@495c0691[cdcr-cluster1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 300442 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/cdcr-cluster1/terms/shard1 to Terms{values={core_node3=0}, version=0}
[junit4] 2> 300450 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.ZkShardTerms Successful update of terms at /collections/cdcr-cluster1/terms/shard2 to Terms{values={core_node4=0}, version=0}
[junit4] 2> 300454 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 300454 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 300454 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:59291/solr/cdcr-cluster1_shard1_replica_n1/
[junit4] 2> 300454 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 300455 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:59291/solr/cdcr-cluster1_shard1_replica_n1/ has no replicas
[junit4] 2> 300455 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 300458 INFO (zkCallback-2160-thread-2) [ ] o.a.s.h.CdcrLeaderStateManager Received new leader state @ cdcr-cluster1:shard1
[junit4] 2> 300465 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:59291/solr/cdcr-cluster1_shard1_replica_n1/ shard1
[junit4] 2> 300468 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 300468 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 300468 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:59291/solr/cdcr-cluster1_shard2_replica_n2/
[junit4] 2> 300468 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 300469 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.SyncStrategy https://127.0.0.1:59291/solr/cdcr-cluster1_shard2_replica_n2/ has no replicas
[junit4] 2> 300469 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 300474 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:59291/solr/cdcr-cluster1_shard2_replica_n2/ shard2
[junit4] 2> 300475 INFO (zkCallback-2160-thread-2) [ ] o.a.s.h.CdcrLeaderStateManager Received new leader state @ cdcr-cluster1:shard2
[junit4] 2> 300619 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 300622 INFO (qtp280651314-5379) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=cdcr-cluster1&newCollection=true&collection=cdcr-cluster1&version=2&replicaType=NRT&coreNodeName=core_node3&name=cdcr-cluster1_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin} status=0 QTime=1528
[junit4] 2> 300627 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 300629 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=cdcr-cluster1&newCollection=true&collection=cdcr-cluster1&version=2&replicaType=NRT&coreNodeName=core_node4&name=cdcr-cluster1_shard2_replica_n2&action=CREATE&numShards=2&shard=shard2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin} status=0 QTime=1534
[junit4] 2> 300644 INFO (qtp280651314-5378) [n:127.0.0.1:59291_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 300730 INFO (zkCallback-2160-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-cluster1/state.json] for collection [cdcr-cluster1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 300774 INFO (OverseerCollectionConfigSetProcessor-72259898820919299-127.0.0.1:59291_solr-n_0000000000) [n:127.0.0.1:59291_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 301644 INFO (qtp280651314-5378) [n:127.0.0.1:59291_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=cdcr-cluster1&maxShardsPerNode=2&name=cdcr-cluster1&nrtReplicas=1&action=CREATE&numShards=2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2} status=0 QTime=2880
[junit4] 2> 301647 INFO (zkConnectionManagerCallback-2184-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 301657 INFO (zkConnectionManagerCallback-2188-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 301659 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[720E9648BD5B3057]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 301660 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[720E9648BD5B3057]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50280/solr ready
[junit4] 2> 301676 INFO (qtp1785814689-5324) [n:127.0.0.1:64026_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=cdcr-cluster2&maxShardsPerNode=2&name=cdcr-cluster2&nrtReplicas=1&action=CREATE&numShards=2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 301681 INFO (OverseerThreadFactory-1464-thread-1-processing-n:127.0.0.1:64026_solr) [n:127.0.0.1:64026_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection cdcr-cluster2
[junit4] 2> 301794 INFO (OverseerStateUpdate-72259898766131203-127.0.0.1:64026_solr-n_0000000000) [n:127.0.0.1:64026_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"cdcr-cluster2",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"cdcr-cluster2_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:64026/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 301799 INFO (OverseerStateUpdate-72259898766131203-127.0.0.1:64026_solr-n_0000000000) [n:127.0.0.1:64026_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"cdcr-cluster2",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"cdcr-cluster2_shard2_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:64026/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 302013 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=cdcr-cluster2&newCollection=true&collection=cdcr-cluster2&version=2&replicaType=NRT&coreNodeName=core_node4&name=cdcr-cluster2_shard2_replica_n2&action=CREATE&numShards=2&shard=shard2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin
[junit4] 2> 302014 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 302018 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=cdcr-cluster2&newCollection=true&collection=cdcr-cluster2&version=2&replicaType=NRT&coreNodeName=core_node3&name=cdcr-cluster2_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin
[junit4] 2> 303038 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.5.0
[junit4] 2> 303038 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.5.0
[junit4] 2> 303059 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.s.IndexSchema [cdcr-cluster2_shard1_replica_n1] Schema name=minimal
[junit4] 2> 303059 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.s.IndexSchema [cdcr-cluster2_shard2_replica_n2] Schema name=minimal
[junit4] 2> 303061 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 303061 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 303061 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'cdcr-cluster2_shard1_replica_n1' using configuration from collection cdcr-cluster2, trusted=true
[junit4] 2> 303061 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'cdcr-cluster2_shard2_replica_n2' using configuration from collection cdcr-cluster2, trusted=true
[junit4] 2> 303061 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_64026.solr.core.cdcr-cluster2.shard2.replica_n2' (registry 'solr.core.cdcr-cluster2.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@17547a5f
[junit4] 2> 303061 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_64026.solr.core.cdcr-cluster2.shard1.replica_n1' (registry 'solr.core.cdcr-cluster2.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@17547a5f
[junit4] 2> 303061 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 303061 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 303062 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.SolrCore [[cdcr-cluster2_shard2_replica_n2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBidirectionalTest_720E9648BD5B3057-001/cdcr-cluster2-001/node1/cdcr-cluster2_shard2_replica_n2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBidirectionalTest_720E9648BD5B3057-001/cdcr-cluster2-001/node1/./cdcr-cluster2_shard2_replica_n2/data/]
[junit4] 2> 303062 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.SolrCore [[cdcr-cluster2_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBidirectionalTest_720E9648BD5B3057-001/cdcr-cluster2-001/node1/cdcr-cluster2_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.cdcr.CdcrBidirectionalTest_720E9648BD5B3057-001/cdcr-cluster2-001/node1/./cdcr-cluster2_shard1_replica_n1/data/]
[junit4] 2> 303228 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
[junit4] 2> 303228 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 303228 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
[junit4] 2> 303228 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 303230 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 303230 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 303230 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 303230 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 303231 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@7a22649f[cdcr-cluster2_shard2_replica_n2] main]
[junit4] 2> 303232 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2783e8a9[cdcr-cluster2_shard1_replica_n1] main]
[junit4] 2> 303233 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/cdcr-cluster2
[junit4] 2> 303234 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/cdcr-cluster2
[junit4] 2> 303234 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/cdcr-cluster2
[junit4] 2> 303234 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/cdcr-cluster2
[junit4] 2> 303234 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/cdcr-cluster2/managed-schema
[junit4] 2> 303234 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/cdcr-cluster2/managed-schema
[junit4] 2> 303235 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 303235 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 303235 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 303236 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 303240 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.CdcrBufferStateManager Created znode /collections/cdcr-cluster2/cdcr/state/buffer
[junit4] 2> 303241 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.CdcrProcessStateManager Created znode /collections/cdcr-cluster2/cdcr/state/process
[junit4] 2> 303244 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1610386825314041856
[junit4] 2> 303244 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1610386825314041856
[junit4] 2> 303246 INFO (searcherExecutor-1494-thread-1-processing-n:127.0.0.1:64026_solr x:cdcr-cluster2_shard1_replica_n1 c:cdcr-cluster2 s:shard1 r:core_node3) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-cluster2_shard1_replica_n1] Registered new searcher Searcher@2783e8a9[cdcr-cluster2_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 303247 INFO (searcherExecutor-1495-thread-1-processing-n:127.0.0.1:64026_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.SolrCore [cdcr-cluster2_shard2_replica_n2] Registered new searcher Searcher@7a22649f[cdcr-cluster2_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 303256 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/cdcr-cluster2/terms/shard1 to Terms{values={core_node3=0}, version=0}
[junit4] 2> 303259 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.ZkShardTerms Successful update of terms at /collections/cdcr-cluster2/terms/shard2 to Terms{values={core_node4=0}, version=0}
[junit4] 2> 303263 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 303263 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 303263 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:64026/solr/cdcr-cluster2_shard1_replica_n1/
[junit4] 2> 303263 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 303264 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:64026/solr/cdcr-cluster2_shard1_replica_n1/ has no replicas
[junit4] 2> 303264 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 303265 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 303265 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 303265 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:64026/solr/cdcr-cluster2_shard2_replica_n2/
[junit4] 2> 303265 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 303266 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.SyncStrategy https://127.0.0.1:64026/solr/cdcr-cluster2_shard2_replica_n2/ has no replicas
[junit4] 2> 303266 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 303268 INFO (zkCallback-2135-thread-2) [ ] o.a.s.h.CdcrLeaderStateManager Received new leader state @ cdcr-cluster2:shard1
[junit4] 2> 303271 INFO (zkCallback-2135-thread-2) [ ] o.a.s.h.CdcrLeaderStateManager Received new leader state @ cdcr-cluster2:shard2
[junit4] 2> 303273 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:64026/solr/cdcr-cluster2_shard1_replica_n1/ shard1
[junit4] 2> 303275 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:64026/solr/cdcr-cluster2_shard2_replica_n2/ shard2
[junit4] 2> 303427 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 303429 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 303433 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=cdcr-cluster2&newCollection=true&collection=cdcr-cluster2&version=2&replicaType=NRT&coreNodeName=core_node3&name=cdcr-cluster2_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin} status=0 QTime=1414
[junit4] 2> 303433 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=cdcr-cluster2&newCollection=true&collection=cdcr-cluster2&version=2&replicaType=NRT&coreNodeName=core_node4&name=cdcr-cluster2_shard2_replica_n2&action=CREATE&numShards=2&shard=shard2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin} status=0 QTime=1419
[junit4] 2> 303438 INFO (qtp1785814689-5324) [n:127.0.0.1:64026_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 303536 INFO (zkCallback-2135-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-cluster2/state.json] for collection [cdcr-cluster2] has occurred - updating... (live nodes size: [1])
[junit4] 2> 303682 INFO (OverseerCollectionConfigSetProcessor-72259898766131203-127.0.0.1:64026_solr-n_0000000000) [n:127.0.0.1:64026_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 304439 INFO (qtp1785814689-5324) [n:127.0.0.1:64026_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=cdcr-cluster2&maxShardsPerNode=2&name=cdcr-cluster2&nrtReplicas=1&action=CREATE&numShards=2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2} status=0 QTime=2763
[junit4] 2> 304447 INFO (zkConnectionManagerCallback-2195-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 304449 INFO (qtp280651314-5381) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 304450 INFO (qtp280651314-5381) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50280/solr ready
[junit4] 2> 304472 INFO (qtp1785814689-5321) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.CdcrRequestHandler Found maxVersionFromRecent 0 maxVersionFromIndex 0
[junit4] 2> 304472 INFO (qtp1785814689-5321) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster2_shard2_replica_n2] webapp=/solr path=/cdcr params={action=SHARDCHECKPOINT&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 304477 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.CdcrRequestHandler Found maxVersionFromRecent 0 maxVersionFromIndex 0
[junit4] 2> 304477 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster2_shard1_replica_n1] webapp=/solr path=/cdcr params={action=SHARDCHECKPOINT&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 304479 INFO (qtp1785814689-5325) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster2_shard2_replica_n2] webapp=/solr path=/cdcr params={_stateVer_=cdcr-cluster2:4&action=COLLECTIONCHECKPOINT&wt=javabin&version=2} status=0 QTime=22
[junit4] 2> 304480 INFO (qtp280651314-5381) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.CdcrReplicatorManager Create new update log reader for target cdcr-cluster2 with checkpoint -1 @ cdcr-cluster1:shard2
[junit4] 2> 304480 INFO (qtp280651314-5381) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.CdcrReplicatorManager Attempting to bootstrap target collection: cdcr-cluster2, shard: shard2
[junit4] 2> 304486 INFO (qtp280651314-5381) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.CdcrReplicatorManager Submitting bootstrap task to executor
[junit4] 2> 304488 INFO (cdcr-bootstrap-status-2191-thread-1-processing-n:127.0.0.1:59291_solr x:cdcr-cluster1_shard2_replica_n2 c:cdcr-cluster1 s:shard2 r:core_node4) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.CdcrReplicatorManager Attempting to bootstrap target collection: cdcr-cluster2 shard: shard2 leader: https://127.0.0.1:64026/solr/cdcr-cluster2_shard2_replica_n2/
[junit4] 2> 304491 INFO (zkCallback-2160-thread-1) [ ] o.a.s.h.CdcrProcessStateManager The CDCR process state has changed: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-cluster1/cdcr/state/process @ cdcr-cluster1:shard2
[junit4] 2> 304491 INFO (zkCallback-2160-thread-2) [ ] o.a.s.h.CdcrProcessStateManager The CDCR process state has changed: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-cluster1/cdcr/state/process @ cdcr-cluster1:shard1
[junit4] 2> 304491 INFO (zkCallback-2160-thread-2) [ ] o.a.s.h.CdcrProcessStateManager Received new CDCR process state from watcher: STARTED @ cdcr-cluster1:shard1
[junit4] 2> 304492 INFO (qtp280651314-5381) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster1_shard2_replica_n2] webapp=/solr path=/cdcr params={qt=/cdcr&_stateVer_=cdcr-cluster1:4&action=start&wt=javabin&version=2} status=0 QTime=48
[junit4] 2> 304493 INFO (zkCallback-2160-thread-1) [ ] o.a.s.h.CdcrProcessStateManager Received new CDCR process state from watcher: STARTED @ cdcr-cluster1:shard2
[junit4] 2> 304494 INFO (zkConnectionManagerCallback-2200-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 304495 INFO (zkCallback-2160-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 304496 INFO (zkCallback-2160-thread-2) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50280/solr ready
[junit4] 2> 304497 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster2_shard2_replica_n2] webapp=/solr path=/cdcr params={qt=/cdcr&masterUrl=https://127.0.0.1:59291/solr/cdcr-cluster1_shard2_replica_n2/&action=BOOTSTRAP&wt=javabin&version=2} status=0 QTime=5
[junit4] 2> 304498 INFO (recoveryExecutor-2132-thread-1-processing-n:127.0.0.1:64026_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 304502 INFO (qtp1785814689-5326) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster2_shard2_replica_n2] webapp=/solr path=/cdcr params={qt=/cdcr&action=BOOTSTRAP_STATUS&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 304502 INFO (cdcr-bootstrap-status-2191-thread-1-processing-n:127.0.0.1:59291_solr x:cdcr-cluster1_shard2_replica_n2 c:cdcr-cluster1 s:shard2 r:core_node4) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.CdcrReplicatorManager CDCR bootstrap running for 1 seconds, sleeping for 2000 ms
[junit4] 2> 304510 INFO (qtp280651314-5381) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1610386826641539072,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 304511 INFO (qtp280651314-5381) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 304513 INFO (qtp280651314-5381) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 304513 INFO (qtp280651314-5381) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster1_shard2_replica_n2] webapp=/solr path=/update params={waitSearcher=true&openSearcher=false&commit=true&softCommit=false&commit_end_point=true&wt=javabin&version=2} status=0 QTime=4
[junit4] 2> 304538 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster1_shard2_replica_n2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 304539 INFO (qtp1785814689-5323) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.CdcrRequestHandler Found maxVersionFromRecent 0 maxVersionFromIndex 0
[junit4] 2> 304539 INFO (qtp1785814689-5323) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster2_shard1_replica_n1] webapp=/solr path=/cdcr params={action=SHARDCHECKPOINT&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 304540 INFO (recoveryExecutor-2132-thread-1-processing-n:127.0.0.1:64026_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 304540 INFO (recoveryExecutor-2132-thread-1-processing-n:127.0.0.1:64026_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 304540 INFO (recoveryExecutor-2132-thread-1-processing-n:127.0.0.1:64026_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 304540 INFO (recoveryExecutor-2132-thread-1-processing-n:127.0.0.1:64026_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 304540 INFO (recoveryExecutor-2132-thread-1-processing-n:127.0.0.1:64026_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 304544 INFO (recoveryExecutor-2132-thread-1-processing-n:127.0.0.1:64026_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@6a210144[cdcr-cluster2_shard2_replica_n2] main]
[junit4] 2> 304545 INFO (searcherExecutor-1495-thread-1-processing-n:127.0.0.1:64026_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.SolrCore [cdcr-cluster2_shard2_replica_n2] Registered new searcher Searcher@6a210144[cdcr-cluster2_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 304547 INFO (recoveryExecutor-2132-thread-1-processing-n:127.0.0.1:64026_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.CdcrRequestHandler No replay needed.
[junit4] 2> 304548 INFO (qtp1785814689-5323) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.CdcrRequestHandler Found maxVersionFromRecent 0 maxVersionFromIndex 0
[junit4] 2> 304548 INFO (qtp1785814689-5323) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster2_shard2_replica_n2] webapp=/solr path=/cdcr params={action=SHARDCHECKPOINT&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 304550 INFO (qtp1785814689-5327) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster2_shard1_replica_n1] webapp=/solr path=/cdcr params={_stateVer_=cdcr-cluster2:4&action=COLLECTIONCHECKPOINT&wt=javabin&version=2} status=0 QTime=34
[junit4] 2> 304550 INFO (zkCallback-2160-thread-2) [ ] o.a.s.h.CdcrReplicatorManager Create new update log reader for target cdcr-cluster2 with checkpoint -1 @ cdcr-cluster1:shard1
[junit4] 2> 304550 INFO (zkCallback-2160-thread-2) [ ] o.a.s.h.CdcrReplicatorManager Attempting to bootstrap target collection: cdcr-cluster2, shard: shard1
[junit4] 2> 304550 INFO (zkCallback-2160-thread-2) [ ] o.a.s.h.CdcrReplicatorManager Submitting bootstrap task to executor
[junit4] 2> 304552 INFO (cdcr-bootstrap-status-2196-thread-1) [ ] o.a.s.h.CdcrReplicatorManager Attempting to bootstrap target collection: cdcr-cluster2 shard: shard1 leader: https://127.0.0.1:64026/solr/cdcr-cluster2_shard1_replica_n1/
[junit4] 2> 304555 INFO (qtp1785814689-5323) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster2_shard1_replica_n1] webapp=/solr path=/cdcr params={qt=/cdcr&masterUrl=https://127.0.0.1:59291/solr/cdcr-cluster1_shard1_replica_n1/&action=BOOTSTRAP&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 304557 INFO (recoveryExecutor-2132-thread-1-processing-n:127.0.0.1:64026_solr x:cdcr-cluster2_shard1_replica_n1 c:cdcr-cluster2 s:shard1 r:core_node3) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 304558 INFO (qtp1785814689-5327) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster2_shard1_replica_n1] webapp=/solr path=/cdcr params={qt=/cdcr&action=BOOTSTRAP_STATUS&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 304558 INFO (cdcr-bootstrap-status-2196-thread-1) [ ] o.a.s.h.CdcrReplicatorManager CDCR bootstrap running for 1 seconds, sleeping for 2000 ms
[junit4] 2> 304563 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1610386826697113600,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 304564 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 304565 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 304565 INFO (qtp280651314-5380) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster1_shard1_replica_n1] webapp=/solr path=/update params={waitSearcher=true&openSearcher=false&commit=true&softCommit=false&commit_end_point=true&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 304568 INFO (qtp280651314-5381) [n:127.0.0.1:59291_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster1_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 304569 INFO (recoveryExecutor-2132-thread-1-processing-n:127.0.0.1:64026_solr x:cdcr-cluster2_shard1_replica_n1 c:cdcr-cluster2 s:shard1 r:core_node3) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 304569 INFO (recoveryExecutor-2132-thread-1-processing-n:127.0.0.1:64026_solr x:cdcr-cluster2_shard1_replica_n1 c:cdcr-cluster2 s:shard1 r:core_node3) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 304569 INFO (recoveryExecutor-2132-thread-1-processing-n:127.0.0.1:64026_solr x:cdcr-cluster2_shard1_replica_n1 c:cdcr-cluster2 s:shard1 r:core_node3) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 304569 INFO (recoveryExecutor-2132-thread-1-processing-n:127.0.0.1:64026_solr x:cdcr-cluster2_shard1_replica_n1 c:cdcr-cluster2 s:shard1 r:core_node3) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 304569 INFO (recoveryExecutor-2132-thread-1-processing-n:127.0.0.1:64026_solr x:cdcr-cluster2_shard1_replica_n1 c:cdcr-cluster2 s:shard1 r:core_node3) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 304572 INFO (recoveryExecutor-2132-thread-1-processing-n:127.0.0.1:64026_solr x:cdcr-cluster2_shard1_replica_n1 c:cdcr-cluster2 s:shard1 r:core_node3) [n:127.0.0.1:64026_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.s
[...truncated too long message...]
ang.Thread.run(Thread.java:748) [?:1.8.0_172]
[junit4] 2> 128324 INFO (Thread-55) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53313 53313
[junit4] 2> 128325 WARN (Thread-55) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/aliases.json
[junit4] 2> 3 /solr/collections/cdcr-cluster2/terms/shard2
[junit4] 2> 3 /solr/collections/cdcr-cluster2/terms/shard1
[junit4] 2> 2 /solr/configs/cdcr-cluster2/managed-schema
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/clusterstate.json
[junit4] 2> 5 /solr/clusterprops.json
[junit4] 2> 4 /solr/collections/cdcr-cluster2/cdcr/state/process
[junit4] 2> 2 /solr/collections/cdcr-cluster2/cdcr/state/buffer
[junit4] 2> 2 /solr/collections/cdcr-cluster2/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/live_nodes
[junit4] 2> 5 /solr/collections
[junit4] 2>
[junit4] 2> 128326 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[720E9648BD5B3057]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testBiDir
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=CdcrBidirectionalTest -Dtests.method=testBiDir -Dtests.seed=720E9648BD5B3057 -Dtests.slow=true -Dtests.badapples=true -Dtests.locale=it-CH -Dtests.timezone=America/Argentina/San_Luis -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 58.7s J0 | CdcrBidirectionalTest.testBiDir <<<
[junit4] > Throwable #1: java.lang.AssertionError: cluster 1 wrong doc expected:<ABC> but was:<null>
[junit4] > at org.apache.solr.cloud.cdcr.CdcrBidirectionalTest.testBiDir(CdcrBidirectionalTest.java:203)
[junit4] > at java.lang.Thread.run(Thread.java:748)Throwable #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=717, name=cdcr-replicator-201-thread-1, state=RUNNABLE, group=TGRP-CdcrBidirectionalTest]
[junit4] > Caused by: java.lang.AssertionError: 1610392463645605888 != 1610392463644557312
[junit4] > at __randomizedtesting.SeedInfo.seed([720E9648BD5B3057]:0)
[junit4] > at org.apache.solr.update.CdcrUpdateLog$CdcrLogReader.forwardSeek(CdcrUpdateLog.java:611)
[junit4] > at org.apache.solr.handler.CdcrReplicator.run(CdcrReplicator.java:105)
[junit4] > at org.apache.solr.handler.CdcrReplicatorScheduler.lambda$null$0(CdcrReplicatorScheduler.java:81)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)Throwable #3: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=726, name=cdcr-replicator-204-thread-1, state=RUNNABLE, group=TGRP-CdcrBidirectionalTest]
[junit4] > Caused by: java.lang.AssertionError: 1610392463645605888 != 1610392463644557312
[junit4] > at __randomizedtesting.SeedInfo.seed([720E9648BD5B3057]:0)
[junit4] > at org.apache.solr.update.CdcrUpdateLog$CdcrLogReader.forwardSeek(CdcrUpdateLog.java:611)
[junit4] > at org.apache.solr.handler.CdcrReplicator.run(CdcrReplicator.java:125)
[junit4] > at org.apache.solr.handler.CdcrReplicatorScheduler.lambda$null$0(CdcrReplicatorScheduler.java:81)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)Throwable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=748, name=cdcr-replicator-201-thread-2, state=RUNNABLE, group=TGRP-CdcrBidirectionalTest]
[junit4] > Caused by: java.lang.AssertionError: 1610392463645605888 != 1610392463644557312
[junit4] > at __randomizedtesting.SeedInfo.seed([720E9648BD5B3057]:0)
[junit4] > at org.apache.solr.update.CdcrUpdateLog$CdcrLogReader.forwardSeek(CdcrUpdateLog.java:611)
[junit4] > at org.apache.solr.handler.CdcrReplicator.run(CdcrReplicator.java:105)
[junit4] > at org.apache.solr.handler.CdcrReplicatorScheduler.lambda$null$0(CdcrReplicatorScheduler.java:81)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)Throwable #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=749, name=cdcr-replicator-204-thread-2, state=RUNNABLE, group=TGRP-CdcrBidirectionalTest]
[junit4] > Caused by: java.lang.AssertionError: 1610392463645605888 != 1610392463644557312
[junit4] > at __randomizedtesting.SeedInfo.seed([720E9648BD5B3057]:0)
[junit4] > at org.apache.solr.update.CdcrUpdateLog$CdcrLogReader.forwardSeek(CdcrUpdateLog.java:611)
[junit4] > at org.apache.solr.handler.CdcrReplicator.run(CdcrReplicator.java:125)
[junit4] > at org.apache.solr.handler.CdcrReplicatorScheduler.lambda$null$0(CdcrReplicatorScheduler.java:81)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.cdcr.CdcrBidirectionalTest_720E9648BD5B3057-001
[junit4] 2> Sep 01, 2018 8:19:26 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 6 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {abc=PostingsFormat(name=LuceneVarGapFixedInterval), xyz=Lucene50(blocksize=128), id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{}, maxPointsInLeafNode=687, maxMBSortInHeap=5.293823728571114, sim=RandomSimilarity(queryNorm=false): {}, locale=it-CH, timezone=America/Argentina/San_Luis
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_172 (64-bit)/cpus=3,threads=1,free=23849472,total=100663296
[junit4] 2> NOTE: All tests run in this JVM: [CdcrBidirectionalTest, CdcrBidirectionalTest, CdcrBidirectionalTest]
[junit4] Completed [10/10 (2!)] on J0 in 59.55s, 1 test, 1 error <<< FAILURES!
[junit4]
[junit4]
[junit4] Tests with failures [seed: 720E9648BD5B3057]:
[junit4] - org.apache.solr.handler.RequestLoggingTest.testLogBeforeExecuteWithRequestLogger
[junit4] - org.apache.solr.cloud.cdcr.CdcrBidirectionalTest.testBiDir
[junit4]
[junit4]
[junit4] JVM J0: 0.67 .. 129.75 = 129.08s
[junit4] JVM J1: 0.71 .. 88.10 = 87.39s
[junit4] Execution time total: 2 minutes 9 seconds
[junit4] Tests summary: 10 suites, 15 tests, 1 error, 1 failure
BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/lucene/common-build.xml:1568: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/lucene/common-build.xml:1092: There were test failures: 10 suites, 15 tests, 1 error, 1 failure [seed: 720E9648BD5B3057]
Total time: 2 minutes 11 seconds
[repro] Setting last failure code to 256
[repro] Failures:
[repro] 1/5 failed: org.apache.solr.cloud.cdcr.CdcrBidirectionalTest
[repro] 1/5 failed: org.apache.solr.handler.RequestLoggingTest
[repro] Exiting with code 256
+ mv lucene/build lucene/build.repro
+ mv solr/build solr/build.repro
+ mv lucene/build.orig lucene/build
+ mv solr/build.orig solr/build
Archiving artifacts
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
[WARNINGS] Parsing warnings in console log with parser Java Compiler (javac)
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
<Git Blamer> Using GitBlamer to create author and commit information for all warnings.
<Git Blamer> GIT_COMMIT=63d4ecd216b8fe21c094a1e4c7da9065569e5077, workspace=/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris
[WARNINGS] Computing warning deltas based on reference build #791
Recording test results
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 794 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/794/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC
3 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSegmentSorting
Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:63303 within 30000 ms
Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:63303 within 30000 ms
at __randomizedtesting.SeedInfo.seed([4DF7C08B6DBE5B6F]:0)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:184)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:121)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:116)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:103)
at org.apache.solr.cloud.MiniSolrCloudCluster.waitForAllNodes(MiniSolrCloudCluster.java:269)
at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:263)
at org.apache.solr.cloud.SolrCloudTestCase$Builder.build(SolrCloudTestCase.java:200)
at org.apache.solr.cloud.SolrCloudTestCase$Builder.configure(SolrCloudTestCase.java:192)
at org.apache.solr.cloud.TestSegmentSorting.setupCluster(TestSegmentSorting.java:57)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:63303 within 30000 ms
at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:232)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:176)
... 32 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSegmentSorting
Error Message:
50 threads leaked from SUITE scope at org.apache.solr.cloud.TestSegmentSorting: 1) Thread[id=15903, name=qtp2039311444-15903, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=15924, name=Scheduler-515515223, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=15867, name=ProcessThread(sid:0 cport:63303):, state=WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:123) 4) Thread[id=15882, name=qtp2111320398-15882, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:292) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:357) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=15906, name=qtp2039311444-15906, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=15864, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-TestSegmentSorting] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:196) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=15886, name=qtp1352908088-15886, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=15910, name=qtp1284264726-15910, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=15883, name=qtp2111320398-15883, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=15884, name=Scheduler-167285512, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=15888, name=qtp1352908088-15888, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=15919, name=Scheduler-1436233543, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=15876, name=qtp2111320398-15876, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=15893, name=Scheduler-1030362953, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=15904, name=qtp2039311444-15904, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=15918, name=Scheduler-493496438, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=15898, name=qtp1473376597-15898, state=RUNNABLE, group=TGRP-TestSegmentSorting] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:396) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$292/1787315108.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=15901, name=qtp2039311444-15901-acceptor-0@3c7ed9f5-ServerConnector@5433c05e{HTTP/1.1,[http/1.1]}{127.0.0.1:41268}, state=RUNNABLE, group=TGRP-TestSegmentSorting] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:369) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:639) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=15894, name=qtp1284264726-15894, state=RUNNABLE, group=TGRP-TestSegmentSorting] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:396) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$292/1787315108.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=15912, name=qtp1473376597-15912, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=15907, name=qtp2039311444-15907, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 22) Thread[id=15902, name=qtp2039311444-15902, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=15914, name=qtp1473376597-15914, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=15880, name=qtp2111320398-15880, state=RUNNABLE, group=TGRP-TestSegmentSorting] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:396) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) 25) Thread[id=15896, name=qtp2039311444-15896, state=RUNNABLE, group=TGRP-TestSegmentSorting] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:396) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$292/1787315108.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=15895, name=qtp1284264726-15895-acceptor-0@2e75ab1a-ServerConnector@262eb4d6{HTTP/1.1,[http/1.1]}{127.0.0.1:39469}, state=RUNNABLE, group=TGRP-TestSegmentSorting] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:369) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:639) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=15892, name=qtp1352908088-15892, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 28) Thread[id=15900, name=qtp1284264726-15900, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 29) Thread[id=15891, name=qtp1352908088-15891, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 30) Thread[id=15897, name=qtp1284264726-15897, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 31) Thread[id=15916, name=qtp1473376597-15916, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 32) Thread[id=15887, name=qtp1352908088-15887-acceptor-0@24be94-ServerConnector@315e8364{HTTP/1.1,[http/1.1]}{127.0.0.1:54662}, state=RUNNABLE, group=TGRP-TestSegmentSorting] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:369) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:639) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) 33) Thread[id=15899, name=qtp1284264726-15899, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 34) Thread[id=15877, name=qtp2111320398-15877, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=15911, name=qtp1473376597-15911-acceptor-0@46492683-ServerConnector@60753650{HTTP/1.1,[http/1.1]}{127.0.0.1:43579}, state=RUNNABLE, group=TGRP-TestSegmentSorting] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:369) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:639) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=15863, name=Thread-3118, state=WAITING, group=TGRP-TestSegmentSorting] at java.lang.Object.wait(Native Method) at java.lang.Thread.join(Thread.java:1252) at java.lang.Thread.join(Thread.java:1326) at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:313) at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:313) at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:496) 37) Thread[id=15865, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at java.lang.Object.wait(Native Method) at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:147) 38) Thread[id=15923, name=qtp1284264726-15923, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 39) Thread[id=15913, name=qtp1473376597-15913, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 40) Thread[id=15915, name=qtp1473376597-15915, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 41) Thread[id=15879, name=qtp2111320398-15879, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 42) Thread[id=15922, name=qtp1284264726-15922, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 43) Thread[id=15889, name=qtp1352908088-15889, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 44) Thread[id=15881, name=qtp2111320398-15881, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 45) Thread[id=15890, name=qtp1352908088-15890, state=RUNNABLE, group=TGRP-TestSegmentSorting] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:396) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at org.eclipse.jetty.io.ManagedSelector$$Lambda$292/1787315108.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) 46) Thread[id=15905, name=qtp2039311444-15905, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 47) Thread[id=15885, name=qtp1352908088-15885, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748) 48) Thread[id=15927, name=Scheduler-1931464837, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 49) Thread[id=15878, name=qtp2111320398-15878-acceptor-0@e818f3e-ServerConnector@444137a6{HTTP/1.1,[http/1.1]}{127.0.0.1:63303}, state=RUNNABLE, group=TGRP-TestSegmentSorting] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:369) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:639) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) 50) Thread[id=15917, name=qtp1473376597-15917, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 50 threads leaked from SUITE scope at org.apache.solr.cloud.TestSegmentSorting:
1) Thread[id=15903, name=qtp2039311444-15903, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=15924, name=Scheduler-515515223, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=15867, name=ProcessThread(sid:0 cport:63303):, state=WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:123)
4) Thread[id=15882, name=qtp2111320398-15882, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:292)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:357)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=15906, name=qtp2039311444-15906, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=15864, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-TestSegmentSorting]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:196)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=15886, name=qtp1352908088-15886, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=15910, name=qtp1284264726-15910, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=15883, name=qtp2111320398-15883, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=15884, name=Scheduler-167285512, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=15888, name=qtp1352908088-15888, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=15919, name=Scheduler-1436233543, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=15876, name=qtp2111320398-15876, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=15893, name=Scheduler-1030362953, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
15) Thread[id=15904, name=qtp2039311444-15904, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
16) Thread[id=15918, name=Scheduler-493496438, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
17) Thread[id=15898, name=qtp1473376597-15898, state=RUNNABLE, group=TGRP-TestSegmentSorting]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:396)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:333)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$292/1787315108.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680)
at java.lang.Thread.run(Thread.java:748)
18) Thread[id=15901, name=qtp2039311444-15901-acceptor-0@3c7ed9f5-ServerConnector@5433c05e{HTTP/1.1,[http/1.1]}{127.0.0.1:41268}, state=RUNNABLE, group=TGRP-TestSegmentSorting]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:369)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:639)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680)
at java.lang.Thread.run(Thread.java:748)
19) Thread[id=15894, name=qtp1284264726-15894, state=RUNNABLE, group=TGRP-TestSegmentSorting]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:396)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:333)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$292/1787315108.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680)
at java.lang.Thread.run(Thread.java:748)
20) Thread[id=15912, name=qtp1473376597-15912, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
21) Thread[id=15907, name=qtp2039311444-15907, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
22) Thread[id=15902, name=qtp2039311444-15902, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
23) Thread[id=15914, name=qtp1473376597-15914, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
24) Thread[id=15880, name=qtp2111320398-15880, state=RUNNABLE, group=TGRP-TestSegmentSorting]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:396)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:333)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680)
at java.lang.Thread.run(Thread.java:748)
25) Thread[id=15896, name=qtp2039311444-15896, state=RUNNABLE, group=TGRP-TestSegmentSorting]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:396)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:333)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$292/1787315108.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680)
at java.lang.Thread.run(Thread.java:748)
26) Thread[id=15895, name=qtp1284264726-15895-acceptor-0@2e75ab1a-ServerConnector@262eb4d6{HTTP/1.1,[http/1.1]}{127.0.0.1:39469}, state=RUNNABLE, group=TGRP-TestSegmentSorting]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:369)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:639)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680)
at java.lang.Thread.run(Thread.java:748)
27) Thread[id=15892, name=qtp1352908088-15892, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
28) Thread[id=15900, name=qtp1284264726-15900, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
29) Thread[id=15891, name=qtp1352908088-15891, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
30) Thread[id=15897, name=qtp1284264726-15897, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
31) Thread[id=15916, name=qtp1473376597-15916, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
32) Thread[id=15887, name=qtp1352908088-15887-acceptor-0@24be94-ServerConnector@315e8364{HTTP/1.1,[http/1.1]}{127.0.0.1:54662}, state=RUNNABLE, group=TGRP-TestSegmentSorting]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:369)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:639)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680)
at java.lang.Thread.run(Thread.java:748)
33) Thread[id=15899, name=qtp1284264726-15899, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
34) Thread[id=15877, name=qtp2111320398-15877, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
35) Thread[id=15911, name=qtp1473376597-15911-acceptor-0@46492683-ServerConnector@60753650{HTTP/1.1,[http/1.1]}{127.0.0.1:43579}, state=RUNNABLE, group=TGRP-TestSegmentSorting]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:369)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:639)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680)
at java.lang.Thread.run(Thread.java:748)
36) Thread[id=15863, name=Thread-3118, state=WAITING, group=TGRP-TestSegmentSorting]
at java.lang.Object.wait(Native Method)
at java.lang.Thread.join(Thread.java:1252)
at java.lang.Thread.join(Thread.java:1326)
at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:313)
at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:313)
at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:496)
37) Thread[id=15865, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at java.lang.Object.wait(Native Method)
at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:147)
38) Thread[id=15923, name=qtp1284264726-15923, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
39) Thread[id=15913, name=qtp1473376597-15913, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
40) Thread[id=15915, name=qtp1473376597-15915, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
41) Thread[id=15879, name=qtp2111320398-15879, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
42) Thread[id=15922, name=qtp1284264726-15922, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
43) Thread[id=15889, name=qtp1352908088-15889, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
44) Thread[id=15881, name=qtp2111320398-15881, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
45) Thread[id=15890, name=qtp1352908088-15890, state=RUNNABLE, group=TGRP-TestSegmentSorting]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:396)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:333)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$292/1787315108.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680)
at java.lang.Thread.run(Thread.java:748)
46) Thread[id=15905, name=qtp2039311444-15905, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
47) Thread[id=15885, name=qtp1352908088-15885, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
48) Thread[id=15927, name=Scheduler-1931464837, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
49) Thread[id=15878, name=qtp2111320398-15878-acceptor-0@e818f3e-ServerConnector@444137a6{HTTP/1.1,[http/1.1]}{127.0.0.1:63303}, state=RUNNABLE, group=TGRP-TestSegmentSorting]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:369)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:639)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680)
at java.lang.Thread.run(Thread.java:748)
50) Thread[id=15917, name=qtp1473376597-15917, state=TIMED_WAITING, group=TGRP-TestSegmentSorting]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:653)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:717)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([4DF7C08B6DBE5B6F]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSegmentSorting
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=15924, name=Scheduler-515515223, state=TIMED_WAITING, group=TGRP-TestSegmentSorting] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=15896, name=qtp2039311444-15896, state=RUNNABLE, group=TGRP-TestSegmentSorting] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.
[...truncated too long message...]
olr-core/classes/test
common-solr.compile-test:
compile-test:
BUILD SUCCESSFUL
Total time: 1 minute 4 seconds
[repro] ant test-nocompile -Dtests.dups=5 -Dtests.maxfailures=5 -Dtests.class="*.TestSegmentSorting" -Dtests.showOutput=onerror "-Dargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Dtests.seed=4DF7C08B6DBE5B6F -Dtests.slow=true -Dtests.locale=ar-MA -Dtests.timezone=IET -Dtests.asserts=true -Dtests.file.encoding=UTF-8
Buildfile: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/build.xml
-clover.disable:
ivy-configure:
[ivy:configure] :: Apache Ivy 2.4.0 - 20141213170938 :: http://ant.apache.org/ivy/ ::
[ivy:configure] :: loading settings :: file = /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/lucene/top-level-ivy-settings.xml
install-junit4-taskdef:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
resolve-groovy:
[ivy:cachepath] :: resolving dependencies :: org.codehaus.groovy#groovy-all-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.codehaus.groovy#groovy-all;2.4.15 in public
[ivy:cachepath] :: resolution report :: resolve 18ms :: artifacts dl 0ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 1 | 0 | 0 | 0 || 1 | 0 |
---------------------------------------------------------------------
-init-totals:
-test:
[mkdir] Created dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test
[junit4:pickseed] Seed property 'tests.seed' already defined: 4DF7C08B6DBE5B6F
[mkdir] Created dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/temp
[junit4] <JUnit4> says aloha! Master seed: 4DF7C08B6DBE5B6F
[junit4] Executing 5 suites with 2 JVMs.
[junit4]
[junit4] Started J1 PID(17315@serv1-vm3).
[junit4] Started J0 PID(17314@serv1-vm3).
[junit4] Suite: org.apache.solr.cloud.TestSegmentSorting
[junit4] OK 23.2s J0 | TestSegmentSorting.testAtomicUpdateOfSegmentSortField
[junit4] OK 6.12s J0 | TestSegmentSorting.testSegmentTerminateEarly
[junit4] Completed [1/5] on J0 in 44.13s, 2 tests
[junit4]
[junit4] Suite: org.apache.solr.cloud.TestSegmentSorting
[junit4] OK 24.3s J1 | TestSegmentSorting.testAtomicUpdateOfSegmentSortField
[junit4] OK 6.10s J1 | TestSegmentSorting.testSegmentTerminateEarly
[junit4] Completed [2/5] on J1 in 44.08s, 2 tests
[junit4]
[junit4] Duplicate suite name used with XML reports: org.apache.solr.cloud.TestSegmentSorting. This may confuse tools that process XML reports. Set 'ignoreDuplicateSuites' to true to skip this message.
[junit4] Suite: org.apache.solr.cloud.TestSegmentSorting
[junit4] OK 15.3s J0 | TestSegmentSorting.testAtomicUpdateOfSegmentSortField
[junit4] OK 6.30s J0 | TestSegmentSorting.testSegmentTerminateEarly
[junit4] Completed [3/5] on J0 in 28.79s, 2 tests
[junit4]
[junit4] Suite: org.apache.solr.cloud.TestSegmentSorting
[junit4] OK 14.5s J1 | TestSegmentSorting.testAtomicUpdateOfSegmentSortField
[junit4] OK 6.25s J1 | TestSegmentSorting.testSegmentTerminateEarly
[junit4] Completed [4/5] on J1 in 28.17s, 2 tests
[junit4]
[junit4] Suite: org.apache.solr.cloud.TestSegmentSorting
[junit4] OK 12.3s J0 | TestSegmentSorting.testAtomicUpdateOfSegmentSortField
[junit4] OK 5.33s J0 | TestSegmentSorting.testSegmentTerminateEarly
[junit4] Completed [5/5] on J0 in 22.94s, 2 tests
[junit4]
[junit4] JVM J0: 0.70 .. 97.32 = 96.61s
[junit4] JVM J1: 0.69 .. 74.54 = 73.85s
[junit4] Execution time total: 1 minute 37 seconds
[junit4] Tests summary: 5 suites, 10 tests
[junit4] Could not remove temporary path: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1 (java.nio.file.DirectoryNotEmptyException: Remaining files: [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp])
[junit4] Could not remove temporary path: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0 (java.nio.file.DirectoryNotEmptyException: Remaining files: [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp])
[echo] 5 slowest tests:
[junit4:tophints] 473.11s | org.apache.solr.cloud.api.collections.ShardSplitTest
[junit4:tophints] 356.37s | org.apache.solr.cloud.cdcr.CdcrWithNodesRestartsTest
[junit4:tophints] 114.38s | org.apache.solr.cloud.TestTlogReplica
[junit4:tophints] 110.73s | org.apache.solr.cloud.ZkFailoverTest
[junit4:tophints] 104.72s | org.apache.solr.cloud.TestPullReplica
-check-totals:
test-nocompile:
BUILD SUCCESSFUL
Total time: 1 minute 39 seconds
[repro] Failures:
[repro] 0/5 failed: org.apache.solr.cloud.TestSegmentSorting
[repro] Exiting with code 0
+ mv lucene/build lucene/build.repro
+ mv solr/build solr/build.repro
+ mv lucene/build.orig lucene/build
+ mv solr/build.orig solr/build
Archiving artifacts
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2846)
at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2725)
at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2706)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3085)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to Solaris VBOX
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1071)
at hudson.FilePath.act(FilePath.java:1060)
at hudson.FilePath.validateAntFileMask(FilePath.java:2704)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1840)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3087)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1073)
at hudson.FilePath.act(FilePath.java:1060)
at hudson.FilePath.validateAntFileMask(FilePath.java:2704)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1840)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/*_pid*.log". Configuration error?
[WARNINGS] Parsing warnings in console log with parser Java Compiler (javac)
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
<Git Blamer> Using GitBlamer to create author and commit information for all warnings.
<Git Blamer> GIT_COMMIT=d35b615c66e8974363b7a938abc0d74a85a798dd, workspace=/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris
[WARNINGS] Computing warning deltas based on reference build #793
Recording test results
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 793 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/793/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC
2 tests failed.
FAILED: org.apache.solr.cloud.api.collections.CollectionsAPIAsyncDistributedZkTest.testAsyncRequests
Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:45021/solr/myalias, http://127.0.0.1:58907/solr/myalias]
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:45021/solr/myalias, http://127.0.0.1:58907/solr/myalias]
at __randomizedtesting.SeedInfo.seed([A022F93B839D68E1:4466C58C2535263E]:0)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:462)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1109)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
at org.apache.solr.cloud.api.collections.CollectionsAPIAsyncDistributedZkTest.testAsyncRequests(CollectionsAPIAsyncDistributedZkTest.java:168)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:58907/solr/myalias: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
<title>Error 404 Can not find: /solr/myalias/select</title>
</head>
<body><h2>HTTP ERROR 404</h2>
<p>Problem accessing /solr/myalias/select. Reason:
<pre> Can not find: /solr/myalias/select</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/>
</body>
</html>
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
... 45 more
FAILED: org.apache.solr.handler.RequestLoggingTest.testLogBeforeExecuteWithRequestLogger
Error Message:
Stack Trace:
java.lang.AssertionError
at __randomizedtesting.SeedInfo.seed([A022F93B839D68E1:114BD9342E6BD0AF]:0)
at org.junit.Assert.fail(Assert.java:92)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertTrue(Assert.java:54)
at org.apache.solr.handler.RequestLoggingTest.testLogBeforeExecute(RequestLoggingTest.java:88)
at org.apache.solr.handler.RequestLoggingTest.testLogBeforeExecuteWithRequestLogger(RequestLoggingTest.java:71)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12347 lines...]
[junit4] Suite: org.apache.solr.cloud.api.collections.CollectionsAPIAsyncDistributedZkTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CollectionsAPIAsyncDistributedZkTest_A022F93B839D68E1-001/init-core-data-001
[junit4] 2> 33460 WARN (SUITE-CollectionsAPIAsyncDistributedZkTest-seed#[A022F93B839D68E1]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=13 numCloses=13
[junit4] 2> 33461 INFO (SUITE-CollectionsAPIAsyncDistributedZkTest-seed#[A022F93B839D68E1]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 33462 INFO (SUITE-CollectionsAPIAsyncDistributedZkTest-seed#[A022F93B839D68E1]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 33462 INFO (SUITE-CollectionsAPIAsyncDistributedZkTest-seed#[A022F93B839D68E1]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 33467 INFO (SUITE-CollectionsAPIAsyncDistributedZkTest-seed#[A022F93B839D68E1]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CollectionsAPIAsyncDistributedZkTest_A022F93B839D68E1-001/tempDir-001
[junit4] 2> 33468 INFO (SUITE-CollectionsAPIAsyncDistributedZkTest-seed#[A022F93B839D68E1]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 33468 INFO (Thread-18) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 33468 INFO (Thread-18) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 33471 ERROR (Thread-18) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 33568 INFO (SUITE-CollectionsAPIAsyncDistributedZkTest-seed#[A022F93B839D68E1]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:48012
[junit4] 2> 33582 INFO (zkConnectionManagerCallback-138-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 33978 INFO (jetty-launcher-135-thread-2) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 33985 INFO (jetty-launcher-135-thread-1) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 34011 INFO (jetty-launcher-135-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 34011 INFO (jetty-launcher-135-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 34014 INFO (jetty-launcher-135-thread-2) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 34039 INFO (jetty-launcher-135-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@26845127{/solr,null,AVAILABLE}
[junit4] 2> 34054 INFO (jetty-launcher-135-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3f4771e3{HTTP/1.1,[http/1.1]}{127.0.0.1:58907}
[junit4] 2> 34054 INFO (jetty-launcher-135-thread-2) [ ] o.e.j.s.Server Started @34145ms
[junit4] 2> 34054 INFO (jetty-launcher-135-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=58907}
[junit4] 2> 34102 INFO (jetty-launcher-135-thread-2) [ ] o.a.s.u.c.SSLCredentialProviderFactory Processing SSL Credential Provider chain: env;sysprop
[junit4] 2> 34107 ERROR (jetty-launcher-135-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 34107 INFO (jetty-launcher-135-thread-2) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 34107 INFO (jetty-launcher-135-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.5.0
[junit4] 2> 34108 INFO (jetty-launcher-135-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 34108 INFO (jetty-launcher-135-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 34108 INFO (jetty-launcher-135-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-09-01T19:04:32.130Z
[junit4] 2> 34110 INFO (jetty-launcher-135-thread-2) [ ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
[junit4] 2> 34111 INFO (jetty-launcher-135-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 34111 INFO (jetty-launcher-135-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 34111 INFO (jetty-launcher-135-thread-1) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 34126 INFO (jetty-launcher-135-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@721e9dbf{/solr,null,AVAILABLE}
[junit4] 2> 34137 INFO (jetty-launcher-135-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@44d41546{HTTP/1.1,[http/1.1]}{127.0.0.1:45021}
[junit4] 2> 34137 INFO (jetty-launcher-135-thread-1) [ ] o.e.j.s.Server Started @34229ms
[junit4] 2> 34137 INFO (jetty-launcher-135-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45021}
[junit4] 2> 34138 ERROR (jetty-launcher-135-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 34138 INFO (jetty-launcher-135-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 34138 INFO (jetty-launcher-135-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.5.0
[junit4] 2> 34138 INFO (jetty-launcher-135-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 34139 INFO (jetty-launcher-135-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 34139 INFO (jetty-launcher-135-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-09-01T19:04:32.161Z
[junit4] 2> 34153 INFO (zkConnectionManagerCallback-140-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 34154 INFO (zkConnectionManagerCallback-142-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 34154 INFO (jetty-launcher-135-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 34155 INFO (jetty-launcher-135-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 34884 INFO (jetty-launcher-135-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48012/solr
[junit4] 2> 34910 INFO (zkConnectionManagerCallback-146-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 34921 INFO (zkConnectionManagerCallback-148-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 34982 INFO (jetty-launcher-135-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48012/solr
[junit4] 2> 35013 INFO (zkConnectionManagerCallback-152-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 35103 INFO (zkConnectionManagerCallback-156-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 35340 INFO (jetty-launcher-135-thread-1) [n:127.0.0.1:45021_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45021_solr
[junit4] 2> 35342 INFO (jetty-launcher-135-thread-1) [n:127.0.0.1:45021_solr ] o.a.s.c.Overseer Overseer (id=72262791838760964-127.0.0.1:45021_solr-n_0000000000) starting
[junit4] 2> 35416 INFO (jetty-launcher-135-thread-2) [n:127.0.0.1:58907_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58907_solr
[junit4] 2> 35451 INFO (zkCallback-155-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 35454 INFO (zkCallback-147-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 35489 INFO (zkConnectionManagerCallback-163-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 35510 INFO (jetty-launcher-135-thread-1) [n:127.0.0.1:45021_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 35513 INFO (jetty-launcher-135-thread-1) [n:127.0.0.1:45021_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48012/solr ready
[junit4] 2> 35565 INFO (zkConnectionManagerCallback-168-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 35567 INFO (jetty-launcher-135-thread-2) [n:127.0.0.1:58907_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 35569 INFO (jetty-launcher-135-thread-2) [n:127.0.0.1:58907_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48012/solr ready
[junit4] 2> 35570 INFO (jetty-launcher-135-thread-2) [n:127.0.0.1:58907_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 35635 INFO (jetty-launcher-135-thread-2) [n:127.0.0.1:58907_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58907.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@52f3ac22
[junit4] 2> 35636 INFO (jetty-launcher-135-thread-2) [n:127.0.0.1:58907_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58907.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@52f3ac22
[junit4] 2> 35636 INFO (jetty-launcher-135-thread-2) [n:127.0.0.1:58907_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58907.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@52f3ac22
[junit4] 2> 35639 INFO (jetty-launcher-135-thread-2) [n:127.0.0.1:58907_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CollectionsAPIAsyncDistributedZkTest_A022F93B839D68E1-001/tempDir-001/node2/.
[junit4] 2> 35649 INFO (OverseerStateUpdate-72262791838760964-127.0.0.1:45021_solr-n_0000000000) [n:127.0.0.1:45021_solr ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:45021_solr
[junit4] 2> 35653 INFO (jetty-launcher-135-thread-1) [n:127.0.0.1:45021_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 35653 INFO (jetty-launcher-135-thread-1) [n:127.0.0.1:45021_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45021_solr
[junit4] 2> 35656 INFO (zkCallback-155-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 35663 INFO (zkCallback-162-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 35665 INFO (zkCallback-167-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 35668 INFO (OverseerStateUpdate-72262791838760964-127.0.0.1:45021_solr-n_0000000000) [n:127.0.0.1:45021_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 35780 INFO (jetty-launcher-135-thread-1) [n:127.0.0.1:45021_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 35818 INFO (jetty-launcher-135-thread-1) [n:127.0.0.1:45021_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45021.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@52f3ac22
[junit4] 2> 35826 INFO (jetty-launcher-135-thread-1) [n:127.0.0.1:45021_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45021.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@52f3ac22
[junit4] 2> 35826 INFO (jetty-launcher-135-thread-1) [n:127.0.0.1:45021_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45021.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@52f3ac22
[junit4] 2> 35827 INFO (jetty-launcher-135-thread-1) [n:127.0.0.1:45021_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CollectionsAPIAsyncDistributedZkTest_A022F93B839D68E1-001/tempDir-001/node1/.
[junit4] 2> 35910 INFO (zkConnectionManagerCallback-172-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 35915 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x100baa056070009, likely client has closed socket
[junit4] 2> 35918 INFO (zkConnectionManagerCallback-177-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 35919 INFO (SUITE-CollectionsAPIAsyncDistributedZkTest-seed#[A022F93B839D68E1]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 35921 INFO (SUITE-CollectionsAPIAsyncDistributedZkTest-seed#[A022F93B839D68E1]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:48012/solr ready
[junit4] IGNOR/A 0.01s J1 | CollectionsAPIAsyncDistributedZkTest.testAsyncIdRaceCondition
[junit4] > Assumption #1: 'badapple' test group is disabled (@BadApple(bugUrl=https://issues.apache.org/jira/browse/SOLR-12028))
[junit4] 2> 35987 INFO (TEST-CollectionsAPIAsyncDistributedZkTest.testAsyncIdBackCompat-seed#[A022F93B839D68E1]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testAsyncIdBackCompat
[junit4] 2> 36425 INFO (qtp74846242-292) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params async=testAsyncIdBackCompat&collection.configName=conf1&name=testAsyncIdBackCompat&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 36440 INFO (qtp74846242-292) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={async=testAsyncIdBackCompat&collection.configName=conf1&name=testAsyncIdBackCompat&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=19
[junit4] 2> 36497 INFO (TEST-CollectionsAPIAsyncDistributedZkTest.testAsyncIdBackCompat-seed#[A022F93B839D68E1]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testAsyncIdBackCompat
[junit4] 2> 36540 INFO (TEST-CollectionsAPIAsyncDistributedZkTest.testSolrJAPICalls-seed#[A022F93B839D68E1]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSolrJAPICalls
[junit4] 2> 36549 INFO (qtp74846242-289) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params async=7a94bb4c-2f3d-4830-bfa0-e92804bd91e6&collection.configName=conf1&name=testasynccollectioncreation&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 36557 INFO (qtp74846242-289) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={async=7a94bb4c-2f3d-4830-bfa0-e92804bd91e6&collection.configName=conf1&name=testasynccollectioncreation&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=8
[junit4] 2> 36573 INFO (qtp74846242-288) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=7a94bb4c-2f3d-4830-bfa0-e92804bd91e6&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 36575 INFO (qtp74846242-288) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=7a94bb4c-2f3d-4830-bfa0-e92804bd91e6&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 36575 INFO (OverseerThreadFactory-35-thread-1-processing-n:127.0.0.1:45021_solr) [n:127.0.0.1:45021_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection testasynccollectioncreation
[junit4] 2> 36725 INFO (OverseerStateUpdate-72262791838760964-127.0.0.1:45021_solr-n_0000000000) [n:127.0.0.1:45021_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testasynccollectioncreation",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"testasynccollectioncreation_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:45021/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 36937 INFO (qtp1397070419-303) [n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=testasynccollectioncreation&version=2&replicaType=NRT&async=7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545&coreNodeName=core_node2&name=testasynccollectioncreation_shard1_replica_n1&action=CREATE&numShards=1&shard=shard1&wt=javabin} status=0 QTime=2
[junit4] 2> 36937 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&collection=testasynccollectioncreation&version=2&replicaType=NRT&async=7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545&coreNodeName=core_node2&name=testasynccollectioncreation_shard1_replica_n1&action=CREATE&numShards=1&shard=shard1&wt=javabin
[junit4] 2> 36943 INFO (qtp1397070419-297) [n:127.0.0.1:45021_solr ] o.a.s.h.a.CoreAdminOperation Checking request status for : 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545
[junit4] 2> 36946 INFO (qtp1397070419-297) [n:127.0.0.1:45021_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&requestid=7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 37579 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=7a94bb4c-2f3d-4830-bfa0-e92804bd91e6&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 37581 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=7a94bb4c-2f3d-4830-bfa0-e92804bd91e6&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 37950 INFO (qtp1397070419-301) [n:127.0.0.1:45021_solr ] o.a.s.h.a.CoreAdminOperation Checking request status for : 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545
[junit4] 2> 37950 INFO (qtp1397070419-301) [n:127.0.0.1:45021_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&requestid=7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 37974 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.5.0
[junit4] 2> 37991 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.s.IndexSchema [testasynccollectioncreation_shard1_replica_n1] Schema name=minimal
[junit4] 2> 37998 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 37998 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'testasynccollectioncreation_shard1_replica_n1' using configuration from collection testasynccollectioncreation, trusted=true
[junit4] 2> 38000 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45021.solr.core.testasynccollectioncreation.shard1.replica_n1' (registry 'solr.core.testasynccollectioncreation.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@52f3ac22
[junit4] 2> 38000 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 38000 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.c.SolrCore [[testasynccollectioncreation_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CollectionsAPIAsyncDistributedZkTest_A022F93B839D68E1-001/tempDir-001/node1/testasynccollectioncreation_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CollectionsAPIAsyncDistributedZkTest_A022F93B839D68E1-001/tempDir-001/node1/./testasynccollectioncreation_shard1_replica_n1/data/]
[junit4] 2> 38092 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 38092 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 38094 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 38094 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 38096 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@25d0fa46[testasynccollectioncreation_shard1_replica_n1] main]
[junit4] 2> 38100 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 38101 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 38102 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 38104 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1610433089897496576
[junit4] 2> 38117 INFO (searcherExecutor-40-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 r:core_node2 CREATE n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.c.SolrCore [testasynccollectioncreation_shard1_replica_n1] Registered new searcher Searcher@25d0fa46[testasynccollectioncreation_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 38125 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/testasynccollectioncreation/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 38137 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 38137 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 38138 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45021/solr/testasynccollectioncreation_shard1_replica_n1/
[junit4] 2> 38140 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 38140 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:45021/solr/testasynccollectioncreation_shard1_replica_n1/ has no replicas
[junit4] 2> 38140 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 38148 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45021/solr/testasynccollectioncreation_shard1_replica_n1/ shard1
[junit4] 2> 38303 INFO (parallelCoreAdminExecutor-36-thread-1-processing-n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545 CREATE) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 38409 INFO (zkCallback-147-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testasynccollectioncreation/state.json] for collection [testasynccollectioncreation] has occurred - updating... (live nodes size: [2])
[junit4] 2> 38584 INFO (qtp74846242-288) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=7a94bb4c-2f3d-4830-bfa0-e92804bd91e6&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 38585 INFO (qtp74846242-288) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=7a94bb4c-2f3d-4830-bfa0-e92804bd91e6&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 38953 INFO (qtp1397070419-300) [n:127.0.0.1:45021_solr ] o.a.s.h.a.CoreAdminOperation Checking request status for : 7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545
[junit4] 2> 38953 INFO (qtp1397070419-300) [n:127.0.0.1:45021_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&requestid=7a94bb4c-2f3d-4830-bfa0-e92804bd91e63131073211176545&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 38974 INFO (OverseerThreadFactory-35-thread-1-processing-n:127.0.0.1:45021_solr) [n:127.0.0.1:45021_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 39588 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=7a94bb4c-2f3d-4830-bfa0-e92804bd91e6&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 39594 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=7a94bb4c-2f3d-4830-bfa0-e92804bd91e6&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=5
[junit4] 2> 39596 INFO (qtp74846242-288) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :deletestatus with params requestid=7a94bb4c-2f3d-4830-bfa0-e92804bd91e6&action=DELETESTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 39598 INFO (qtp74846242-288) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=7a94bb4c-2f3d-4830-bfa0-e92804bd91e6&action=DELETESTATUS&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 39600 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params async=469dec96-6e6d-4fe2-8f76-fc0585860568&collection.configName=conf1&name=testasynccollectioncreation&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 39605 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={async=469dec96-6e6d-4fe2-8f76-fc0585860568&collection.configName=conf1&name=testasynccollectioncreation&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=4
[junit4] 2> 39607 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=469dec96-6e6d-4fe2-8f76-fc0585860568&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 39609 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=469dec96-6e6d-4fe2-8f76-fc0585860568&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 39615 INFO (OverseerThreadFactory-35-thread-2-processing-n:127.0.0.1:45021_solr) [n:127.0.0.1:45021_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection testasynccollectioncreation
[junit4] 2> 39615 INFO (OverseerCollectionConfigSetProcessor-72262791838760964-127.0.0.1:45021_solr-n_0000000000) [n:127.0.0.1:45021_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 39617 ERROR (OverseerThreadFactory-35-thread-2-processing-n:127.0.0.1:45021_solr) [n:127.0.0.1:45021_solr ] o.a.s.c.a.c.OverseerCollectionMessageHandler Collection: testasynccollectioncreation operation: create failed:org.apache.solr.common.SolrException: collection already exists: testasynccollectioncreation
[junit4] 2> at org.apache.solr.cloud.api.collections.CreateCollectionCmd.call(CreateCollectionCmd.java:110)
[junit4] 2> at org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:259)
[junit4] 2> at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:478)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2>
[junit4] 2> 39628 INFO (OverseerThreadFactory-35-thread-2-processing-n:127.0.0.1:45021_solr) [n:127.0.0.1:45021_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000001 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 40612 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=469dec96-6e6d-4fe2-8f76-fc0585860568&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 40615 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=469dec96-6e6d-4fe2-8f76-fc0585860568&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 40619 INFO (qtp74846242-288) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :deletestatus with params requestid=469dec96-6e6d-4fe2-8f76-fc0585860568&action=DELETESTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 40620 INFO (qtp74846242-288) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=469dec96-6e6d-4fe2-8f76-fc0585860568&action=DELETESTATUS&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 40623 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params async=be8132fb-2e66-48a9-b277-6a3901e75b04&action=ADDREPLICA&collection=testasynccollectioncreation&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 40627 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr c:testasynccollectioncreation ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={async=be8132fb-2e66-48a9-b277-6a3901e75b04&action=ADDREPLICA&collection=testasynccollectioncreation&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=4
[junit4] 2> 40647 INFO (OverseerCollectionConfigSetProcessor-72262791838760964-127.0.0.1:45021_solr-n_0000000000) [n:127.0.0.1:45021_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000001 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 40657 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=be8132fb-2e66-48a9-b277-6a3901e75b04&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 40658 INFO (OverseerThreadFactory-35-thread-3-processing-n:127.0.0.1:45021_solr) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:58907_solr for creating new replica of shard shard1
[junit4] 2> 40659 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=be8132fb-2e66-48a9-b277-6a3901e75b04&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 40664 INFO (OverseerStateUpdate-72262791838760964-127.0.0.1:45021_solr-n_0000000000) [n:127.0.0.1:45021_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"addreplica",
[junit4] 2> "collection":"testasynccollectioncreation",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"testasynccollectioncreation_shard1_replica_n3",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:58907/solr",
[junit4] 2> "node_name":"127.0.0.1:58907_solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 40669 INFO (zkCallback-147-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testasynccollectioncreation/state.json] for collection [testasynccollectioncreation] has occurred - updating... (live nodes size: [2])
[junit4] 2> 40770 INFO (qtp74846242-288) [n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={async=be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865&qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf1&name=testasynccollectioncreation_shard1_replica_n3&action=CREATE&collection=testasynccollectioncreation&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=0
[junit4] 2> 40770 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3] o.a.s.h.a.CoreAdminOperation core create command async=be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865&qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf1&name=testasynccollectioncreation_shard1_replica_n3&action=CREATE&collection=testasynccollectioncreation&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 40771 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 40773 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CoreAdminOperation Checking request status for : be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865
[junit4] 2> 40773 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&requestid=be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 40793 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.5.0
[junit4] 2> 40816 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.s.IndexSchema [testasynccollectioncreation_shard1_replica_n3] Schema name=minimal
[junit4] 2> 40820 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 40821 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.CoreContainer Creating SolrCore 'testasynccollectioncreation_shard1_replica_n3' using configuration from collection testasynccollectioncreation, trusted=true
[junit4] 2> 40821 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58907.solr.core.testasynccollectioncreation.shard1.replica_n3' (registry 'solr.core.testasynccollectioncreation.shard1.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@52f3ac22
[junit4] 2> 40821 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 40821 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.SolrCore [[testasynccollectioncreation_shard1_replica_n3] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CollectionsAPIAsyncDistributedZkTest_A022F93B839D68E1-001/tempDir-001/node2/testasynccollectioncreation_shard1_replica_n3], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.CollectionsAPIAsyncDistributedZkTest_A022F93B839D68E1-001/tempDir-001/node2/./testasynccollectioncreation_shard1_replica_n3/data/]
[junit4] 2> 40888 INFO (zkCallback-147-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testasynccollectioncreation/state.json] for collection [testasynccollectioncreation] has occurred - updating... (live nodes size: [2])
[junit4] 2> 40944 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 40944 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 40946 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 40946 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 40947 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.s.SolrIndexSearcher Opening [Searcher@7792d1bf[testasynccollectioncreation_shard1_replica_n3] main]
[junit4] 2> 40952 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 40954 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 40955 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 40955 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1610433092886986752
[junit4] 2> 40958 INFO (searcherExecutor-45-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.SolrCore [testasynccollectioncreation_shard1_replica_n3] Registered new searcher Searcher@7792d1bf[testasynccollectioncreation_shard1_replica_n3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 40965 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.ZkShardTerms Successful update of terms at /collections/testasynccollectioncreation/terms/shard1 to Terms{values={core_node2=0, core_node4=0}, version=1}
[junit4] 2> 40970 INFO (parallelCoreAdminExecutor-29-thread-1-processing-n:127.0.0.1:58907_solr x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 CREATE) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.ZkController Core needs to recover:testasynccollectioncreation_shard1_replica_n3
[junit4] 2> 40973 INFO (updateExecutor-149-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 40984 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 40985 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 41006 INFO (qtp1397070419-301) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.c.S.Request [testasynccollectioncreation_shard1_replica_n1] webapp=/solr path=/admin/ping params={wt=javabin&version=2} hits=0 status=0 QTime=15
[junit4] 2> 41006 INFO (qtp1397070419-301) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.c.S.Request [testasynccollectioncreation_shard1_replica_n1] webapp=/solr path=/admin/ping params={wt=javabin&version=2} status=0 QTime=16
[junit4] 2> 41008 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[testasynccollectioncreation_shard1_replica_n3]
[junit4] 2> 41008 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 41009 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.RecoveryStrategy Publishing state of core [testasynccollectioncreation_shard1_replica_n3] as recovering, leader is [http://127.0.0.1:45021/solr/testasynccollectioncreation_shard1_replica_n1/] and I am [http://127.0.0.1:58907/solr/testasynccollectioncreation_shard1_replica_n3/]
[junit4] 2> 41016 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.ZkShardTerms Successful update of terms at /collections/testasynccollectioncreation/terms/shard1 to Terms{values={core_node2=0, core_node4_recovering=0, core_node4=0}, version=2}
[junit4] 2> 41022 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:45021/solr]; [WaitForState: action=PREPRECOVERY&core=testasynccollectioncreation_shard1_replica_n1&nodeName=127.0.0.1:58907_solr&coreNodeName=core_node4&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 41030 INFO (qtp1397070419-301) [n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node4, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true, maxTime: 183 s
[junit4] 2> 41032 INFO (qtp1397070419-301) [n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=testasynccollectioncreation, shard=shard1, thisCore=testasynccollectioncreation_shard1_replica_n1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:58907_solr, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{"core":"testasynccollectioncreation_shard1_replica_n3","base_url":"http://127.0.0.1:58907/solr","node_name":"127.0.0.1:58907_solr","state":"down","type":"NRT","force_set_state":"false"}
[junit4] 2> 41139 INFO (zkCallback-155-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testasynccollectioncreation/state.json] for collection [testasynccollectioncreation] has occurred - updating... (live nodes size: [2])
[junit4] 2> 41140 INFO (zkCallback-155-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testasynccollectioncreation/state.json] for collection [testasynccollectioncreation] has occurred - updating... (live nodes size: [2])
[junit4] 2> 41140 INFO (zkCallback-147-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testasynccollectioncreation/state.json] for collection [testasynccollectioncreation] has occurred - updating... (live nodes size: [2])
[junit4] 2> 41671 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=be8132fb-2e66-48a9-b277-6a3901e75b04&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 41673 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=be8132fb-2e66-48a9-b277-6a3901e75b04&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 41778 INFO (qtp74846242-288) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CoreAdminOperation Checking request status for : be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865
[junit4] 2> 41778 INFO (qtp74846242-288) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&requestid=be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 41783 INFO (OverseerThreadFactory-35-thread-3-processing-n:127.0.0.1:45021_solr) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 42033 INFO (qtp1397070419-301) [n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=testasynccollectioncreation, shard=shard1, thisCore=testasynccollectioncreation_shard1_replica_n1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:58907_solr, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{"core":"testasynccollectioncreation_shard1_replica_n3","base_url":"http://127.0.0.1:58907/solr","node_name":"127.0.0.1:58907_solr","state":"recovering","type":"NRT","force_set_state":"false"}
[junit4] 2> 42033 INFO (qtp1397070419-301) [n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node4, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 42033 INFO (qtp1397070419-301) [n:127.0.0.1:45021_solr x:testasynccollectioncreation_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:58907_solr&onlyIfLeaderActive=true&core=testasynccollectioncreation_shard1_replica_n1&coreNodeName=core_node4&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1008
[junit4] 2> 42535 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:45021/solr/testasynccollectioncreation_shard1_replica_n1/] - recoveringAfterStartup=[true]
[junit4] 2> 42542 WARN (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.u.PeerSyncWithLeader no frame of reference to tell if we've missed updates
[junit4] 2> 42542 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.RecoveryStrategy PeerSync Recovery was not successful - trying replication.
[junit4] 2> 42542 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.RecoveryStrategy Starting Replication Recovery.
[junit4] 2> 42542 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.RecoveryStrategy Attempting to replicate from [http://127.0.0.1:45021/solr/testasynccollectioncreation_shard1_replica_n1/].
[junit4] 2> 42554 INFO (qtp1397070419-301) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1610433094563659776,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 42554 INFO (qtp1397070419-301) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 42555 INFO (qtp1397070419-301) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 42556 INFO (qtp1397070419-301) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [testasynccollectioncreation_shard1_replica_n1] webapp=/solr path=/update params={waitSearcher=true&openSearcher=false&commit=true&softCommit=false&commit_end_point=true&wt=javabin&version=2}{commit=} 0 8
[junit4] 2> 42575 INFO (qtp1397070419-300) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 r:core_node2 x:testasynccollectioncreation_shard1_replica_n1] o.a.s.c.S.Request [testasynccollectioncreation_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 42575 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 42576 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 42576 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 42576 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 42576 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 42578 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.s.SolrIndexSearcher Opening [Searcher@11365c91[testasynccollectioncreation_shard1_replica_n3] main]
[junit4] 2> 42581 INFO (searcherExecutor-45-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.SolrCore [testasynccollectioncreation_shard1_replica_n3] Registered new searcher Searcher@11365c91[testasynccollectioncreation_shard1_replica_n3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 42582 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 42582 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.RecoveryStrategy Replication Recovery was successful.
[junit4] 2> 42582 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 42584 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.ZkShardTerms Successful update of terms at /collections/testasynccollectioncreation/terms/shard1 to Terms{values={core_node2=0, core_node4=0}, version=3}
[junit4] 2> 42584 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.RecoveryStrategy Updating version bucket highest from index after successful recovery.
[junit4] 2> 42584 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1610433094595117056
[junit4] 2> 42601 INFO (recoveryExecutor-150-thread-1-processing-x:testasynccollectioncreation_shard1_replica_n3 be8132fb-2e66-48a9-b277-6a3901e75b043131077257964865 r:core_node4 CREATE n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1) [n:127.0.0.1:58907_solr c:testasynccollectioncreation s:shard1 r:core_node4 x:testasynccollectioncreation_shard1_replica_n3] o.a.s.c.RecoveryStrategy Finished recovery process, successful=[true]
[junit4] 2> 42677 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=be8132fb-2e66-48a9-b277-6a3901e75b04&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 42680 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=be8132fb-2e66-48a9-b277-6a3901e75b04&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 42682 INFO (qtp74846242-288) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :deletestatus with params requestid=be8132fb-2e66-48a9-b277-6a3901e75b04&action=DELETESTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 42684 INFO (qtp74846242-288) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=be8132fb-2e66-48a9-b277-6a3901e75b04&action=DELETESTATUS&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 42686 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :splitshard with params async=46dc2749-9633-4b0c-ae96-ea10e226419d&action=SPLITSHARD&collection=testasynccollectioncreation&shard=shard1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 42689 INFO (OverseerCollectionConfigSetProcessor-72262791838760964-127.0.0.1:45021_solr-n_0000000000) [n:127.0.0.1:45021_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 42692 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr c:testasynccollectioncreation ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={async=46dc2749-9633-4b0c-ae96-ea10e226419d&action=SPLITSHARD&collection=testasynccollectioncreation&shard=shard1&wt=javabin&version=2} status=0 QTime=6
[junit4] 2> 42694 INFO (qtp74846242-288) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=46dc2749-9633-4b0c-ae96-ea10e226419d&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 42695 INFO (zkCallback-147-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testasynccollectioncreation/state.json] for collection [testasynccollectioncreation] has occurred - updating... (live nodes size: [2])
[junit4] 2> 42697 INFO (zkCallback-155-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testasynccollectioncreation/state.json] for collection [testasynccollectioncreation] has occurred - updating... (live nodes size: [2])
[junit4] 2> 42697 INFO (zkCallback-155-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testasynccollectioncreation/state.json] for collection [testasynccollectioncreation] has occurred - updating... (live nodes size: [2])
[junit4] 2> 42701 INFO (qtp74846242-288) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=46dc2749-9633-4b0c-ae96-ea10e226419d&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=7
[junit4] 2> 42750 INFO (qtp1397070419-300) [n:127.0.0.1:45021_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace&wt=javabin&version=2&group=solr.node} status=0 QTime=14
[junit4] 2> 42761 INFO (qtp1397070419-301) [n:127.0.0.1:45021_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.testasynccollectioncreation.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=1
[junit4] 2> 42893 INFO (zkCallback-155-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testasynccollectioncreation/state.json] for collection [testasynccollectioncreation] has occurred - updating... (live nodes size: [2])
[junit4] 2> 42894 INFO (zkCallback-147-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testasynccollectioncreation/state.json] for collection [testasynccollectioncreation] has occurred - updating... (live nodes size: [2])
[junit4] 2> 42895 INFO (zkCallback-155-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testasynccollectioncreation/state.json] for collection [testasynccollectioncreation] has occurred - updating... (live nodes size: [2])
[junit4] 2> 43705 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=46dc2749-9633-4b0c-ae96-ea10e226419d&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 43706 INFO (qtp74846242-294) [n:127.0.0.1:58907_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=46dc2749-9633-4b0c-ae96-ea10e226419d&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 43779 INFO (OverseerThreadFactory-35-thread-4-processing-n:127.0.0.1:45021_solr) [n:127.0.0.1:45021_solr c:testasynccollectioncreation s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:45021_solr for creating new replica of shard shard1_0
[junit4] 2> 43782 INFO (OverseerStateUpdate-72262791838760964-127.0.0.1:45021_solr-n_0000000000) [n:127.0.0.1:45021_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"addreplica",
[junit4] 2> "collection":"testasynccollectioncreation",
[junit4] 2> "shard":"shard1_0",
[junit4] 2> "core":"testasynccollectioncreation_shard1_0_replica_n5",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:45021/solr",
[junit4] 2> "node_name":"127.0.0.1:45021_solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 43886 INFO (zkCallback-155-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testasynccollectioncreation/stat
[...truncated too long message...]
rJAPICalls
[junit4] OK 22.7s J1 | CollectionsAPIAsyncDistributedZkTest.testAsyncRequests
[junit4] Completed [2/10] on J1 in 50.72s, 4 tests
[junit4]
[junit4] Duplicate suite name used with XML reports: org.apache.solr.cloud.api.collections.CollectionsAPIAsyncDistributedZkTest. This may confuse tools that process XML reports. Set 'ignoreDuplicateSuites' to true to skip this message.
[junit4] Suite: org.apache.solr.handler.RequestLoggingTest
[junit4] OK 0.02s J1 | RequestLoggingTest.testLogBeforeExecuteWithCoreLogger
[junit4] OK 0.01s J1 | RequestLoggingTest.testLogBeforeExecuteWithRequestLogger
[junit4] Completed [3/10] on J1 in 2.07s, 2 tests
[junit4]
[junit4] Suite: org.apache.solr.cloud.api.collections.CollectionsAPIAsyncDistributedZkTest
[junit4] OK 3.08s J0 | CollectionsAPIAsyncDistributedZkTest.testAsyncIdRaceCondition
[junit4] OK 0.04s J0 | CollectionsAPIAsyncDistributedZkTest.testAsyncIdBackCompat
[junit4] OK 16.2s J0 | CollectionsAPIAsyncDistributedZkTest.testSolrJAPICalls
[junit4] OK 22.3s J0 | CollectionsAPIAsyncDistributedZkTest.testAsyncRequests
[junit4] Completed [4/10] on J0 in 44.95s, 4 tests
[junit4]
[junit4] Suite: org.apache.solr.cloud.api.collections.CollectionsAPIAsyncDistributedZkTest
[junit4] OK 3.08s J1 | CollectionsAPIAsyncDistributedZkTest.testAsyncIdRaceCondition
[junit4] OK 0.03s J1 | CollectionsAPIAsyncDistributedZkTest.testAsyncIdBackCompat
[junit4] OK 15.2s J1 | CollectionsAPIAsyncDistributedZkTest.testSolrJAPICalls
[junit4] OK 22.4s J1 | CollectionsAPIAsyncDistributedZkTest.testAsyncRequests
[junit4] Completed [5/10] on J1 in 43.11s, 4 tests
[junit4]
[junit4] Suite: org.apache.solr.handler.RequestLoggingTest
[junit4] OK 0.02s J1 | RequestLoggingTest.testLogBeforeExecuteWithCoreLogger
[junit4] OK 0.01s J1 | RequestLoggingTest.testLogBeforeExecuteWithRequestLogger
[junit4] Completed [6/10] on J1 in 1.03s, 2 tests
[junit4]
[junit4] Duplicate suite name used with XML reports: org.apache.solr.handler.RequestLoggingTest. This may confuse tools that process XML reports. Set 'ignoreDuplicateSuites' to true to skip this message.
[junit4] Suite: org.apache.solr.handler.RequestLoggingTest
[junit4] OK 0.01s J1 | RequestLoggingTest.testLogBeforeExecuteWithCoreLogger
[junit4] OK 0.00s J1 | RequestLoggingTest.testLogBeforeExecuteWithRequestLogger
[junit4] Completed [7/10] on J1 in 0.89s, 2 tests
[junit4]
[junit4] Suite: org.apache.solr.handler.RequestLoggingTest
[junit4] OK 0.01s J1 | RequestLoggingTest.testLogBeforeExecuteWithCoreLogger
[junit4] OK 0.01s J1 | RequestLoggingTest.testLogBeforeExecuteWithRequestLogger
[junit4] Completed [8/10] on J1 in 1.07s, 2 tests
[junit4]
[junit4] Suite: org.apache.solr.handler.RequestLoggingTest
[junit4] OK 0.01s J1 | RequestLoggingTest.testLogBeforeExecuteWithCoreLogger
[junit4] OK 0.01s J1 | RequestLoggingTest.testLogBeforeExecuteWithRequestLogger
[junit4] Completed [9/10] on J1 in 0.90s, 2 tests
[junit4]
[junit4] Suite: org.apache.solr.cloud.api.collections.CollectionsAPIAsyncDistributedZkTest
[junit4] OK 3.09s J0 | CollectionsAPIAsyncDistributedZkTest.testAsyncIdRaceCondition
[junit4] OK 0.06s J0 | CollectionsAPIAsyncDistributedZkTest.testAsyncIdBackCompat
[junit4] OK 15.1s J0 | CollectionsAPIAsyncDistributedZkTest.testSolrJAPICalls
[junit4] OK 22.3s J0 | CollectionsAPIAsyncDistributedZkTest.testAsyncRequests
[junit4] Completed [10/10] on J0 in 44.07s, 4 tests
[junit4]
[junit4] JVM J0: 0.69 .. 139.43 = 138.74s
[junit4] JVM J1: 0.69 .. 101.55 = 100.86s
[junit4] Execution time total: 2 minutes 19 seconds
[junit4] Tests summary: 10 suites, 30 tests
[junit4] Could not remove temporary path: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1 (java.nio.file.DirectoryNotEmptyException: Remaining files: [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp])
[junit4] Could not remove temporary path: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0 (java.nio.file.DirectoryNotEmptyException: Remaining files: [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp])
[echo] 5 slowest tests:
[junit4:tophints] 453.38s | org.apache.solr.cloud.api.collections.ShardSplitTest
[junit4:tophints] 344.33s | org.apache.solr.cloud.cdcr.CdcrWithNodesRestartsTest
[junit4:tophints] 263.62s | org.apache.solr.cloud.cdcr.CdcrOpsAndBoundariesTest
[junit4:tophints] 130.93s | org.apache.solr.cloud.autoscaling.IndexSizeTriggerTest
[junit4:tophints] 118.07s | org.apache.solr.cloud.autoscaling.SearchRateTriggerIntegrationTest
-check-totals:
test-nocompile:
BUILD SUCCESSFUL
Total time: 2 minutes 21 seconds
[repro] Failures:
[repro] 0/5 failed: org.apache.solr.cloud.api.collections.CollectionsAPIAsyncDistributedZkTest
[repro] 0/5 failed: org.apache.solr.handler.RequestLoggingTest
[repro] Exiting with code 0
+ mv lucene/build lucene/build.repro
+ mv solr/build solr/build.repro
+ mv lucene/build.orig lucene/build
+ mv solr/build.orig solr/build
Archiving artifacts
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2846)
at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2725)
at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2706)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3085)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to Solaris VBOX
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1071)
at hudson.FilePath.act(FilePath.java:1060)
at hudson.FilePath.validateAntFileMask(FilePath.java:2704)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1840)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3087)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1073)
at hudson.FilePath.act(FilePath.java:1060)
at hudson.FilePath.validateAntFileMask(FilePath.java:2704)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1840)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/*_pid*.log". Configuration error?
[WARNINGS] Parsing warnings in console log with parser Java Compiler (javac)
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
<Git Blamer> Using GitBlamer to create author and commit information for all warnings.
<Git Blamer> GIT_COMMIT=d35b615c66e8974363b7a938abc0d74a85a798dd, workspace=/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris
[WARNINGS] Computing warning deltas based on reference build #792
Recording test results
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2