You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2018/03/15 16:09:56 UTC
[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build #
1734 - Failure!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1734/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseG1GC
1 tests failed.
FAILED: org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test
Error Message:
Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: { "control_collection":{ "pullReplicas":"0", "replicationFactor":"1", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{"core_node2":{ "core":"control_collection_shard1_replica_n1", "base_url":"http://127.0.0.1:62648/dl/ew", "node_name":"127.0.0.1:62648_dl%2Few", "state":"active", "type":"NRT", "leader":"true"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"1", "tlogReplicas":"0"}, "collection1":{ "pullReplicas":"0", "replicationFactor":"1", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{ "core_node22":{ "core":"collection1_shard1_replica_t21", "base_url":"http://127.0.0.1:52399/dl/ew", "node_name":"127.0.0.1:52399_dl%2Few", "state":"active", "type":"TLOG", "leader":"true"}, "core_node24":{ "core":"collection1_shard1_replica_t23", "base_url":"http://127.0.0.1:64943/dl/ew", "node_name":"127.0.0.1:64943_dl%2Few", "state":"recovering", "type":"TLOG"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"1", "tlogReplicas":"0"}}
Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {
"control_collection":{
"pullReplicas":"0",
"replicationFactor":"1",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{"core_node2":{
"core":"control_collection_shard1_replica_n1",
"base_url":"http://127.0.0.1:62648/dl/ew",
"node_name":"127.0.0.1:62648_dl%2Few",
"state":"active",
"type":"NRT",
"leader":"true"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"1",
"tlogReplicas":"0"},
"collection1":{
"pullReplicas":"0",
"replicationFactor":"1",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{
"core_node22":{
"core":"collection1_shard1_replica_t21",
"base_url":"http://127.0.0.1:52399/dl/ew",
"node_name":"127.0.0.1:52399_dl%2Few",
"state":"active",
"type":"TLOG",
"leader":"true"},
"core_node24":{
"core":"collection1_shard1_replica_t23",
"base_url":"http://127.0.0.1:64943/dl/ew",
"node_name":"127.0.0.1:64943_dl%2Few",
"state":"recovering",
"type":"TLOG"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"1",
"tlogReplicas":"0"}}
at __randomizedtesting.SeedInfo.seed([95A7A0DC3B52F81B:1DF39F0695AE95E3]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2005)
at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:116)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 1846 lines...]
[junit4] JVM J0: stdout was not empty, see: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/build/core/test/temp/junit4-J0-20180315_143733_9541180553243220927029.sysout
[junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
[junit4] codec: Asserting, pf: FSTOrd50, dvf: Lucene70
[junit4] <<< JVM J0: EOF ----
[...truncated 12180 lines...]
[junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/init-core-data-001
[junit4] 2> 2538330 WARN (SUITE-RecoveryAfterSoftCommitTest-seed#[95A7A0DC3B52F81B]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=6 numCloses=6
[junit4] 2> 2538331 INFO (SUITE-RecoveryAfterSoftCommitTest-seed#[95A7A0DC3B52F81B]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 2538332 INFO (SUITE-RecoveryAfterSoftCommitTest-seed#[95A7A0DC3B52F81B]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
[junit4] 2> 2538332 INFO (SUITE-RecoveryAfterSoftCommitTest-seed#[95A7A0DC3B52F81B]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 2538332 INFO (SUITE-RecoveryAfterSoftCommitTest-seed#[95A7A0DC3B52F81B]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /dl/ew
[junit4] 2> 2538334 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2538334 INFO (Thread-5019) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2538335 INFO (Thread-5019) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2538337 ERROR (Thread-5019) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2538434 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.ZkTestServer start zk server on port:54034
[junit4] 2> 2538453 INFO (zkConnectionManagerCallback-5320-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2538466 INFO (zkConnectionManagerCallback-5322-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2538475 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 2538517 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 2538521 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 2538523 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 2538525 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 2538526 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 2538528 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 2538530 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 2538531 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 2538533 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 2538534 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 2538536 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
[junit4] 2> 2538677 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T16:27:37-05:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 2538678 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2538679 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2538679 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 2538679 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@39bea704{/dl/ew,null,AVAILABLE}
[junit4] 2> 2538679 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3fda0bc9{HTTP/1.1,[http/1.1]}{127.0.0.1:50816}
[junit4] 2> 2538679 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.Server Started @2542206ms
[junit4] 2> 2538679 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/tempDir-001/control/data, replicaType=NRT, hostContext=/dl/ew, hostPort=62648, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/control-001/cores}
[junit4] 2> 2538680 ERROR (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2538680 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 2538680 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2538680 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2538680 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-15T15:36:55.209Z
[junit4] 2> 2538682 INFO (zkConnectionManagerCallback-5324-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2538683 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2538683 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/control-001/solr.xml
[junit4] 2> 2538686 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2538686 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2538687 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2538689 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54034/solr
[junit4] 2> 2538691 INFO (zkConnectionManagerCallback-5328-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2538694 INFO (zkConnectionManagerCallback-5330-thread-1-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2538855 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2538858 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:62648_dl%2Few
[junit4] 2> 2538863 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.Overseer Overseer (id=72064476004220932-127.0.0.1:62648_dl%2Few-n_0000000000) starting
[junit4] 2> 2538873 INFO (zkConnectionManagerCallback-5335-thread-1-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2538875 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:54034/solr ready
[junit4] 2> 2538877 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62648_dl%2Few
[junit4] 2> 2538881 INFO (zkCallback-5329-thread-1-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2538882 INFO (zkCallback-5334-thread-1-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2539075 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:62648_dl%2Few ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
[junit4] 2> 2539082 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:62648_dl%2Few ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
[junit4] 2> 2539082 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:62648_dl%2Few ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
[junit4] 2> 2539083 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/control-001/cores
[junit4] 2> 2539104 INFO (zkConnectionManagerCallback-5339-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2539106 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2539108 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:54034/solr ready
[junit4] 2> 2539108 INFO (SocketProxy-Acceptor-62648) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=46834,localport=62648], receiveBufferSize:130880
[junit4] 2> 2539109 INFO (SocketProxy-Acceptor-62648) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=50816,localport=45296], receiveBufferSize=139264
[junit4] 2> 2539110 INFO (qtp686307169-24869) [n:127.0.0.1:62648_dl%2Few ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:62648_dl%252Few&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2539113 INFO (OverseerThreadFactory-9898-thread-1-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 2539223 INFO (SocketProxy-Acceptor-62648) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=36484,localport=62648], receiveBufferSize:130880
[junit4] 2> 2539224 INFO (SocketProxy-Acceptor-62648) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=50816,localport=39435], receiveBufferSize=139264
[junit4] 2> 2539224 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2539225 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2539332 INFO (zkCallback-5329-thread-1-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2540256 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 2540279 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 2540420 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2540466 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 2540467 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
[junit4] 2> 2540467 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2540467 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 2540470 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=4.8076171875, floorSegmentMB=1.9384765625, forceMergeDeletesPctAllowed=22.775887864813843, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.31703850642037046
[junit4] 2> 2540474 WARN (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2540528 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2540528 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=2 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2540529 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2540529 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2540530 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=22, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 2540530 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@22a6a530[control_collection_shard1_replica_n1] main]
[junit4] 2> 2540536 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2540537 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2540537 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2540538 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1595018541339246592
[junit4] 2> 2540540 INFO (searcherExecutor-9901-thread-1-processing-n:127.0.0.1:62648_dl%2Few x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@22a6a530[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2540550 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 2540556 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2540556 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2540556 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:62648/dl/ew/control_collection_shard1_replica_n1/
[junit4] 2> 2540557 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2540557 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:62648/dl/ew/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 2540557 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2540566 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:62648/dl/ew/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 2540672 INFO (zkCallback-5329-thread-2-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2540723 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2540732 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1508
[junit4] 2> 2540748 INFO (qtp686307169-24869) [n:127.0.0.1:62648_dl%2Few ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2540831 INFO (zkCallback-5329-thread-2-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2541122 INFO (OverseerCollectionConfigSetProcessor-72064476004220932-127.0.0.1:62648_dl%2Few-n_0000000000) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2541749 INFO (qtp686307169-24869) [n:127.0.0.1:62648_dl%2Few ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:62648_dl%252Few&wt=javabin&version=2} status=0 QTime=2639
[junit4] 2> 2541754 INFO (zkConnectionManagerCallback-5343-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2541756 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2541777 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:54034/solr ready
[junit4] 2> 2541777 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 2541777 INFO (SocketProxy-Acceptor-62648) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=62448,localport=62648], receiveBufferSize:130880
[junit4] 2> 2541778 INFO (SocketProxy-Acceptor-62648) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=50816,localport=36628], receiveBufferSize=139264
[junit4] 2> 2541778 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2541786 INFO (OverseerThreadFactory-9898-thread-2-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 2541788 WARN (OverseerThreadFactory-9898-thread-2-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 2542061 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2542063 INFO (qtp686307169-24873) [n:127.0.0.1:62648_dl%2Few ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=284
[junit4] 2> 2542398 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-1-001 of type TLOG
[junit4] 2> 2542399 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T16:27:37-05:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 2542399 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2542399 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2542399 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 2542400 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1483ff35{/dl/ew,null,AVAILABLE}
[junit4] 2> 2542400 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@559d3cc4{HTTP/1.1,[http/1.1]}{127.0.0.1:46791}
[junit4] 2> 2542400 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.Server Started @2545928ms
[junit4] 2> 2542400 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/dl/ew, hostPort=52399, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-1-001/cores}
[junit4] 2> 2542401 ERROR (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2542401 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 2542401 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2542401 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2542401 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-15T15:36:58.930Z
[junit4] 2> 2542422 INFO (zkConnectionManagerCallback-5345-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2542429 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2542429 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-1-001/solr.xml
[junit4] 2> 2542433 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2542433 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2542434 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2542449 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54034/solr
[junit4] 2> 2542451 INFO (zkConnectionManagerCallback-5349-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2542459 INFO (zkConnectionManagerCallback-5351-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2542486 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:52399_dl%2Few ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2542492 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:52399_dl%2Few ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2542500 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:52399_dl%2Few ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52399_dl%2Few
[junit4] 2> 2542508 INFO (zkCallback-5329-thread-2-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2542509 INFO (zkCallback-5334-thread-1-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2542513 INFO (zkCallback-5342-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2542528 INFO (zkCallback-5350-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2542699 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:52399_dl%2Few ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
[junit4] 2> 2542712 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:52399_dl%2Few ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
[junit4] 2> 2542712 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:52399_dl%2Few ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
[junit4] 2> 2542714 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:52399_dl%2Few ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-1-001/cores
[junit4] 2> 2542773 INFO (zkConnectionManagerCallback-5356-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2542774 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:52399_dl%2Few ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2542776 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:52399_dl%2Few ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:54034/solr ready
[junit4] 2> 2542858 INFO (SocketProxy-Acceptor-52399) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=35601,localport=52399], receiveBufferSize:130880
[junit4] 2> 2542858 INFO (SocketProxy-Acceptor-52399) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46791,localport=54110], receiveBufferSize=139264
[junit4] 2> 2542860 INFO (qtp1572325424-24928) [n:127.0.0.1:52399_dl%2Few ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:52399_dl%252Few&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2542881 INFO (OverseerCollectionConfigSetProcessor-72064476004220932-127.0.0.1:62648_dl%2Few-n_0000000000) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2542883 INFO (OverseerThreadFactory-9898-thread-3-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:52399_dl%2Few for creating new replica
[junit4] 2> 2542905 INFO (SocketProxy-Acceptor-52399) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=52806,localport=52399], receiveBufferSize:130880
[junit4] 2> 2542909 INFO (SocketProxy-Acceptor-52399) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46791,localport=34758], receiveBufferSize=139264
[junit4] 2> 2542910 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 2542910 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2543031 INFO (zkCallback-5350-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2543961 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 2543978 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.s.IndexSchema [collection1_shard1_replica_t21] Schema name=test
[junit4] 2> 2544101 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2544127 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t21' using configuration from collection collection1, trusted=true
[junit4] 2> 2544128 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t21' (registry 'solr.core.collection1.shard1.replica_t21') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
[junit4] 2> 2544128 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2544128 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.c.SolrCore [[collection1_shard1_replica_t21] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-1-001/cores/collection1_shard1_replica_t21], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-1-001/cores/collection1_shard1_replica_t21/data/]
[junit4] 2> 2544130 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=4.8076171875, floorSegmentMB=1.9384765625, forceMergeDeletesPctAllowed=22.775887864813843, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.31703850642037046
[junit4] 2> 2544132 WARN (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2544187 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2544187 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=2 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2544188 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2544188 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2544189 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=22, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 2544189 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.s.SolrIndexSearcher Opening [Searcher@1766ac70[collection1_shard1_replica_t21] main]
[junit4] 2> 2544191 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2544192 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2544193 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2544193 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1595018545171791872
[junit4] 2> 2544194 INFO (searcherExecutor-9912-thread-1-processing-n:127.0.0.1:52399_dl%2Few x:collection1_shard1_replica_t21 s:shard1 c:collection1) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.c.SolrCore [collection1_shard1_replica_t21] Registered new searcher Searcher@1766ac70[collection1_shard1_replica_t21] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2544200 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard1 to Terms{values={core_node22=0}, version=0}
[junit4] 2> 2544206 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2544206 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2544206 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:52399/dl/ew/collection1_shard1_replica_t21/
[junit4] 2> 2544206 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2544206 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.c.SyncStrategy http://127.0.0.1:52399/dl/ew/collection1_shard1_replica_t21/ has no replicas
[junit4] 2> 2544206 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2544207 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.c.ZkController collection1_shard1_replica_t21 stopping background replication from leader
[junit4] 2> 2544213 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:52399/dl/ew/collection1_shard1_replica_t21/ shard1
[junit4] 2> 2544320 INFO (zkCallback-5350-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2544371 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2544374 INFO (qtp1572325424-24939) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t21] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1464
[junit4] 2> 2544378 INFO (qtp1572325424-24928) [n:127.0.0.1:52399_dl%2Few ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:52399_dl%252Few&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=1518
[junit4] 2> 2544475 INFO (zkCallback-5350-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2544554 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-2-001 of type TLOG
[junit4] 2> 2544555 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T16:27:37-05:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 2544563 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2544564 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2544564 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 2544582 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7c562958{/dl/ew,null,AVAILABLE}
[junit4] 2> 2544583 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4f5da30a{HTTP/1.1,[http/1.1]}{127.0.0.1:60058}
[junit4] 2> 2544583 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.e.j.s.Server Started @2548111ms
[junit4] 2> 2544583 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/dl/ew, hostPort=64943, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-2-001/cores}
[junit4] 2> 2544583 ERROR (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2544584 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 2544584 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2544584 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2544584 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-15T15:37:01.113Z
[junit4] 2> 2544587 INFO (zkConnectionManagerCallback-5358-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2544589 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2544589 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-2-001/solr.xml
[junit4] 2> 2544591 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2544592 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2544593 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2544595 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54034/solr
[junit4] 2> 2544603 INFO (zkConnectionManagerCallback-5362-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2544606 INFO (zkConnectionManagerCallback-5364-thread-1-processing-n:127.0.0.1:64943_dl%2Few) [n:127.0.0.1:64943_dl%2Few ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2544625 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:64943_dl%2Few ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2544634 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:64943_dl%2Few ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2544638 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:64943_dl%2Few ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64943_dl%2Few
[junit4] 2> 2544640 INFO (zkCallback-5334-thread-1-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2544640 INFO (zkCallback-5342-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2544640 INFO (zkCallback-5350-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2544641 INFO (zkCallback-5329-thread-2-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2544642 INFO (zkCallback-5355-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2544645 INFO (zkCallback-5363-thread-1-processing-n:127.0.0.1:64943_dl%2Few) [n:127.0.0.1:64943_dl%2Few ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2544767 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:64943_dl%2Few ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
[junit4] 2> 2544780 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:64943_dl%2Few ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
[junit4] 2> 2544780 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:64943_dl%2Few ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
[junit4] 2> 2544781 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:64943_dl%2Few ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-2-001/cores
[junit4] 2> 2544789 INFO (zkConnectionManagerCallback-5369-thread-1-processing-n:127.0.0.1:64943_dl%2Few) [n:127.0.0.1:64943_dl%2Few ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2544791 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:64943_dl%2Few ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2544793 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [n:127.0.0.1:64943_dl%2Few ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:54034/solr ready
[junit4] 2> 2544842 INFO (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:64943_dl%252Few&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2544845 INFO (OverseerCollectionConfigSetProcessor-72064476004220932-127.0.0.1:62648_dl%2Few-n_0000000000) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2544847 INFO (OverseerThreadFactory-9898-thread-4-processing-n:127.0.0.1:62648_dl%2Few) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:64943_dl%2Few for creating new replica
[junit4] 2> 2544852 INFO (SocketProxy-Acceptor-64943) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=36438,localport=64943], receiveBufferSize:130880
[junit4] 2> 2544853 INFO (SocketProxy-Acceptor-64943) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=60058,localport=54002], receiveBufferSize=139264
[junit4] 2> 2544853 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t23&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 2544854 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2544960 INFO (zkCallback-5350-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2544961 INFO (zkCallback-5363-thread-1-processing-n:127.0.0.1:64943_dl%2Few) [n:127.0.0.1:64943_dl%2Few ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2545874 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 2545884 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.s.IndexSchema [collection1_shard1_replica_t23] Schema name=test
[junit4] 2> 2545965 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2545985 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t23' using configuration from collection collection1, trusted=true
[junit4] 2> 2545985 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t23' (registry 'solr.core.collection1.shard1.replica_t23') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@b81b1b4
[junit4] 2> 2545985 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2545986 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.c.SolrCore [[collection1_shard1_replica_t23] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-2-001/cores/collection1_shard1_replica_t23], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest_95A7A0DC3B52F81B-001/shard-2-001/cores/collection1_shard1_replica_t23/data/]
[junit4] 2> 2545987 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=4.8076171875, floorSegmentMB=1.9384765625, forceMergeDeletesPctAllowed=22.775887864813843, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.31703850642037046
[junit4] 2> 2545989 WARN (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2546026 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2546026 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=2 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2546027 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2546027 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2546028 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=22, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 2546028 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.s.SolrIndexSearcher Opening [Searcher@78d81afa[collection1_shard1_replica_t23] main]
[junit4] 2> 2546029 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2546030 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2546030 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2546031 INFO (searcherExecutor-9923-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.c.SolrCore [collection1_shard1_replica_t23] Registered new searcher Searcher@78d81afa[collection1_shard1_replica_t23] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2546031 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1595018547099074560
[junit4] 2> 2546035 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard1 to Terms{values={core_node24=0, core_node22=0}, version=1}
[junit4] 2> 2546037 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.c.ZkController Core needs to recover:collection1_shard1_replica_t23
[junit4] 2> 2546037 INFO (updateExecutor-5359-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2546037 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2546037 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2546037 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.ZkController collection1_shard1_replica_t23 stopping background replication from leader
[junit4] 2> 2546037 INFO (qtp1397148686-24971) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 x:collection1_shard1_replica_t23] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t23&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1184
[junit4] 2> 2546042 INFO (SocketProxy-Acceptor-52399) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=53634,localport=52399], receiveBufferSize:130880
[junit4] 2> 2546043 INFO (SocketProxy-Acceptor-52399) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46791,localport=48703], receiveBufferSize=139264
[junit4] 2> 2546043 INFO (qtp1572325424-24932) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_t21] o.a.s.c.S.Request [collection1_shard1_replica_t21] webapp=/dl/ew path=/admin/ping params={wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2546043 INFO (qtp1572325424-24932) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_t21] o.a.s.c.S.Request [collection1_shard1_replica_t21] webapp=/dl/ew path=/admin/ping params={wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2546044 INFO (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:64943_dl%252Few&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=1202
[junit4] 2> 2546044 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard1_replica_t23]
[junit4] 2> 2546044 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2546044 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard1_replica_t23] as recovering, leader is [http://127.0.0.1:52399/dl/ew/collection1_shard1_replica_t21/] and I am [http://127.0.0.1:64943/dl/ew/collection1_shard1_replica_t23/]
[junit4] 2> 2546046 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard1 to Terms{values={core_node24_recovering=0, core_node24=0, core_node22=0}, version=2}
[junit4] 2> 2546048 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:52399/dl/ew]; [WaitForState: action=PREPRECOVERY&core=collection1_shard1_replica_t21&nodeName=127.0.0.1:64943_dl%252Few&coreNodeName=core_node24&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2546048 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 2546048 INFO (TEST-RecoveryAfterSoftCommitTest.test-seed#[95A7A0DC3B52F81B]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):330
[junit4] 1> -
[junit4] 2> 2546049 INFO (SocketProxy-Acceptor-52399) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=55334,localport=52399], receiveBufferSize:130880
[junit4] 2> 2546049 INFO (SocketProxy-Acceptor-52399) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46791,localport=62663], receiveBufferSize=139264
[junit4] 1> replica:core_node22 rstate:active live:true
[junit4] 1> replica:core_node24 rstate:down live:true
[junit4] 2> 2546050 INFO (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node24, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true, maxTime: 183 s
[junit4] 2> 2546050 INFO (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1_shard1_replica_t21, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:64943_dl%2Few, coreNodeName=core_node24, onlyIfActiveCheckResult=false, nodeProps: core_node24:{"core":"collection1_shard1_replica_t23","base_url":"http://127.0.0.1:64943/dl/ew","node_name":"127.0.0.1:64943_dl%2Few","state":"down","type":"TLOG"}
[junit4] 2> 2546149 INFO (zkCallback-5363-thread-1-processing-n:127.0.0.1:64943_dl%2Few) [n:127.0.0.1:64943_dl%2Few ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2546149 INFO (zkCallback-5350-thread-1-processing-n:127.0.0.1:52399_dl%2Few) [n:127.0.0.1:52399_dl%2Few ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2546848 INFO (OverseerCollectionConfigSetProcessor-72064476004220932-127.0.0.1:62648_dl%2Few-n_0000000000) [n:127.0.0.1:62648_dl%2Few ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 1> -
[junit4] 1> replica:core_node22 rstate:active live:true
[junit4] 1> replica:core_node24 rstate:recovering live:true
[junit4] 2> 2547051 INFO (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1_shard1_replica_t21, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:64943_dl%2Few, coreNodeName=core_node24, onlyIfActiveCheckResult=false, nodeProps: core_node24:{"core":"collection1_shard1_replica_t23","base_url":"http://127.0.0.1:64943/dl/ew","node_name":"127.0.0.1:64943_dl%2Few","state":"recovering","type":"TLOG"}
[junit4] 2> 2547051 INFO (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node24, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 2547051 INFO (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:64943_dl%252Few&onlyIfLeaderActive=true&core=collection1_shard1_replica_t21&coreNodeName=core_node24&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1002
[junit4] 2> 2547553 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.RecoveryStrategy Starting Replication Recovery.
[junit4] 2> 2547553 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.RecoveryStrategy Attempting to replicate from [http://127.0.0.1:52399/dl/ew/collection1_shard1_replica_t21/].
[junit4] 2> 2547554 INFO (SocketProxy-Acceptor-52399) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=39896,localport=52399], receiveBufferSize:130880
[junit4] 2> 2547555 INFO (SocketProxy-Acceptor-52399) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46791,localport=39551], receiveBufferSize=139264
[junit4] 2> 2547555 INFO (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_t21] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1595018548697104384,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2547555 INFO (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_t21] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2547555 INFO (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_t21] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2547555 INFO (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_t21] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t21] webapp=/dl/ew path=/update params={waitSearcher=true&openSearcher=false&commit=true&softCommit=false&commit_end_point=true&wt=javabin&version=2}{commit=} 0 0
[junit4] 2> 2547557 INFO (SocketProxy-Acceptor-52399) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=56444,localport=52399], receiveBufferSize:130880
[junit4] 2> 2547557 INFO (SocketProxy-Acceptor-52399) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46791,localport=53659], receiveBufferSize=139264
[junit4] 2> 2547558 INFO (qtp1572325424-24940) [n:127.0.0.1:52399_dl%2Few c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_t21] o.a.s.c.S.Request [collection1_shard1_replica_t21] webapp=/dl/ew path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 2547558 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 2547558 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 2547558 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 2547558 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 2547559 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 2547559 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2547559 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@4b5e13f8 commitCommandVersion:0
[junit4] 2> 2547559 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.s.SolrIndexSearcher Opening [Searcher@6fb8fcf6[collection1_shard1_replica_t23] main]
[junit4] 2> 2547559 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2547561 INFO (searcherExecutor-9923-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.SolrCore [collection1_shard1_replica_t23] Registered new searcher Searcher@6fb8fcf6[collection1_shard1_replica_t23] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2547561 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.RecoveryStrategy Replication Recovery was successful.
[junit4] 2> 2547562 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_shard1_replica_t23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:64943_dl%2Few c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_t23] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2547562 INFO (recoveryExecutor-5360-thread-1-processing-n:127.0.0.1:64943_dl%2Few x:collection1_
[...truncated too long message...]
est, TestReplicationHandlerBackup, TestCloudManagedSchema, DistribJoinFromCollectionTest, AliasIntegrationTest, TestCollationFieldDocValues, TestLegacyTerms, SolrXmlInZkTest, TestHighlightDedupGrouping, TestDelegationWithHadoopAuth, RAMDirectoryFactoryTest, HdfsWriteToMultipleCollectionsTest, TestUseDocValuesAsStored2, FacetPivotSmallTest, TestDFISimilarityFactory, ZkShardTermsTest, ConvertedLegacyTest, CollectionsAPISolrJTest, TestMiniSolrCloudClusterSSL, TestJsonFacets, TestCloudConsistency, TestManagedResourceStorage, TestRequestForwarding, HttpSolrCallGetCoreTest, TriggerIntegrationTest, HttpPartitionOnCommitTest, TestIndexSearcher, TestXmlQParser, CoreSorterTest, PolyFieldTest, TestSweetSpotSimilarityFactory, TestTlogReplica, MoveReplicaHDFSTest, RuleEngineTest, FastVectorHighlighterTest, TestConfigsApi, SolrMetricReporterTest, CurrencyFieldTypeTest, DirectUpdateHandlerOptimizeTest, SpatialRPTFieldTypeTest, TestSolrCloudWithDelegationTokens, TestCollationField, JvmMetricsTest, SpatialHeatmapFacetsTest, TestClusterStateMutator, TestSchemaManager, TestSizeLimitedDistributedMap, TestGraphMLResponseWriter, LeaderElectionTest, TestFieldSortValues, DistributedFacetPivotSmallAdvancedTest, TestExactSharedStatsCache, TestLRUStatsCache, TestPseudoReturnFields, TestFaceting, TestFieldCacheReopen, TestSerializedLuceneMatchVersion, DirectSolrSpellCheckerTest, SortSpecParsingTest, LIROnShardRestartTest, SchemaVersionSpecificBehaviorTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, LukeRequestHandlerTest, HdfsSyncSliceTest, TestReRankQParserPlugin, TestRangeQuery, TestScoreJoinQPNoScore, LeaderFailureAfterFreshStartTest, SimpleCollectionCreateDeleteTest, TestLuceneIndexBackCompat, TestJettySolrRunner, SchemaApiFailureTest, BlockJoinFacetDistribTest, SolrPluginUtilsTest, TermVectorComponentDistributedTest, DistributedMLTComponentTest, TestReloadAndDeleteDocs, VMParamsZkACLAndCredentialsProvidersTest, UpdateLogTest, SolrIndexSplitterTest, NumericFieldsTest, NumberUtilsTest, AlternateDirectoryTest, TestSurroundQueryParser, SystemLogListenerTest, TestLegacyFieldReuse, StatelessScriptUpdateProcessorFactoryTest, TestConfigOverlay, TestLFUCache, MetricUtilsTest, AtomicUpdatesTest, AddSchemaFieldsUpdateProcessorFactoryTest, AsyncCallRequestStatusResponseTest, TestDocTermOrds, TestDistribDocBasedVersion, ReplicationFactorTest, TestCharFilters, CustomHighlightComponentTest, TestDownShardTolerantSearch, TestTolerantSearch, ClassificationUpdateProcessorIntegrationTest, SpellingQueryConverterTest, TestCSVResponseWriter, TestNumericTokenStream, TestBM25SimilarityFactory, HighlighterTest, TestSQLHandlerNonCloud, TestCryptoKeys, AnalyticsQueryTest, DistributedQueryElevationComponentTest, ZkNodePropsTest, TestFieldTypeCollectionResource, TestBinaryField, TestQueryUtils, HdfsChaosMonkeyNothingIsSafeTest, AnalysisErrorHandlingTest, TestRandomCollapseQParserPlugin, CursorMarkTest, NodeLostTriggerTest, TestPerFieldSimilarityWithDefaultOverride, TestRandomRequestDistribution, ConfigSetsAPITest, TestInPlaceUpdatesDistrib, SynonymTokenizerTest, CloneFieldUpdateProcessorFactoryTest, DistributedQueryComponentOptimizationTest, BasicZkTest, ResourceLoaderTest, TestSubQueryTransformer, TestCoreBackup, TestMaxScoreQueryParser, TemplateUpdateProcessorTest, UUIDUpdateProcessorFallbackTest, TestExportWriter, TestPivotHelperCode, TestMinMaxOnMultiValuedField, TestSolrFieldCacheBean, TestLeaderElectionWithEmptyReplica, BigEndianAscendingWordDeserializerTest, TestZkChroot, UniqFieldsUpdateProcessorFactoryTest, TestConfigSetProperties, TestStressUserVersions, PingRequestHandlerTest, MBeansHandlerTest, TestTolerantUpdateProcessorRandomCloud, QueryEqualityTest, ConcurrentDeleteAndCreateCollectionTest, DistributedVersionInfoTest, TestJmxIntegration, BaseCdcrDistributedZkTest, SolrShardReporterTest, TestOverriddenPrefixQueryForCustomFieldType, CdcrReplicationHandlerTest, TestPostingsSolrHighlighter, CollectionReloadTest, TermsComponentTest, TestReloadDeadlock, FileUtilsTest, TestComplexPhraseQParserPlugin, QueryElevationComponentTest, TestMacroExpander, CachingDirectoryFactoryTest, TestLargeCluster, JSONWriterTest, TestImplicitCoreProperties, TestFileDictionaryLookup, BasicDistributedZkTest, SimpleFacetsTest, SolrCoreTest, TestGroupingSearch, SolrCmdDistributorTest, PeerSyncTest, DirectUpdateHandlerTest, TestBadConfig, SimplePostToolTest, AnalysisAfterCoreReloadTest, SignatureUpdateProcessorFactoryTest, CoreAdminHandlerTest, SolrRequestParserTest, DocValuesTest, SuggesterTSTTest, SuggesterTest, TestStressLucene, TestTrie, SpatialFilterTest, SuggesterWFSTTest, TestCSVLoader, WordBreakSolrSpellCheckerTest, FieldMutatingUpdateProcessorTest, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, DefaultValueUpdateProcessorTest, TestRemoteStreaming, DocValuesMultiTest, TestSolrDeletionPolicy1, SolrInfoBeanTest, XsltUpdateRequestHandlerTest, CacheHeaderTest, DocumentAnalysisRequestHandlerTest, TestQueryTypes, TestOmitPositions, XmlUpdateRequestHandlerTest, RequestHandlersTest, DocumentBuilderTest, PathHierarchyTokenizerFactoryTest, TestIndexingPerformance, MoreLikeThisHandlerTest, RequiredFieldsTest, TestSolrQueryParser, IndexSchemaRuntimeFieldTest, UpdateRequestProcessorFactoryTest, JsonLoaderTest, BinaryUpdateRequestHandlerTest, TestComponentsName, SearchHandlerTest, SOLR749Test, TestQuerySenderListener, UpdateParamsTest, CopyFieldTest, ResponseLogComponentTest, SolrIndexConfigTest, BadComponentTest, TestMergePolicyConfig, MultiTermTest, SampleTest, TestDocSet, MinimalSchemaTest, TestElisionMultitermQuery, OutputWriterTest, TestSolrCoreProperties, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestPhraseSuggestions, SpellPossibilityIteratorTest, TestCodecSupport, TestXIncludeConfig, TestDFRSimilarityFactory, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestIBSimilarityFactory, TestFastWriter, TestFastOutputStream, OpenExchangeRatesOrgProviderTest, URLClassifyProcessorTest, PluginInfoTest, DateMathParserTest, PreAnalyzedFieldTest, TestSystemIdResolver, PrimUtilsTest, TestSuggestSpellingConverter, DOMUtilTest, TestSolrJ, TestLRUCache, TestUtils, SliceStateTest, TestRTGBase, CursorPagingTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestSimpleTrackingShardHandler, AssignBackwardCompatibilityTest, ConnectionManagerTest, CreateCollectionCleanupTest, DeleteNodeTest, DeleteStatusTest, DistributedQueueTest, DocValuesNotIndexedTest, ForceLeaderTest, HealthCheckHandlerTest, HttpPartitionTest, LeaderElectionContextKeyTest, MissingSegmentRecoveryTest, MultiThreadedOCPTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerModifyCollectionTest, RecoveryAfterSoftCommitTest]
[junit4] Completed [580/785 (1!)] on J0 in 642.29s, 1 test, 1 failure <<< FAILURES!
[...truncated 36779 lines...]
-ecj-javadoc-lint-src:
[mkdir] Created dir: /var/tmp/ecj343962066
[ecj-lint] Compiling 827 source files to /var/tmp/ecj343962066
[ecj-lint] ----------
[ecj-lint] 1. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java (at line 523)
[ecj-lint] throw new CorruptIndexException("misplaced codec footer (file truncated?): length=" + in.length() + " but footerLength==" + footerLength(), input);
[ecj-lint] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[ecj-lint] Resource leak: 'in' is not closed at this location
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 2. ERROR in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (at line 738)
[ecj-lint] } finally {
[ecj-lint] throw t;
[ecj-lint] }
[ecj-lint] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[ecj-lint] finally block does not complete normally
[ecj-lint] ----------
[ecj-lint] ----------
[ecj-lint] 3. WARNING in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java (at line 841)
[ecj-lint] SegmentReader newReader = new SegmentReader(info, reader, liveDocs, info.info.maxDoc() - delCount);
[ecj-lint] ^^^^^^^^^
[ecj-lint] Resource leak: 'newReader' is never closed
[ecj-lint] ----------
[ecj-lint] 3 problems (1 error, 2 warnings)
BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:618: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:101: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/build.xml:203: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/common-build.xml:2082: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/common-build.xml:2121: Compile failed; see the compiler error output for details.
Total time: 92 minutes 32 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build #
1736 - Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1736/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC
5 tests failed.
FAILED: org.apache.solr.cloud.autoscaling.ScheduledTriggerTest.testTrigger
Error Message:
Did not expect the listener to fire on first run!
Stack Trace:
java.lang.AssertionError: Did not expect the listener to fire on first run!
at __randomizedtesting.SeedInfo.seed([16B355D9DBFA6A65:7578635B42351948]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.autoscaling.ScheduledTriggerTest.lambda$new$0(ScheduledTriggerTest.java:48)
at org.apache.solr.cloud.autoscaling.ScheduledTrigger.run(ScheduledTrigger.java:191)
at org.apache.solr.cloud.autoscaling.ScheduledTriggerTest.scheduledTriggerTest(ScheduledTriggerTest.java:102)
at org.apache.solr.cloud.autoscaling.ScheduledTriggerTest.testTrigger(ScheduledTriggerTest.java:75)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.core.TestLazyCores
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([16B355D9DBFA6A65]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.core.TestLazyCores
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([16B355D9DBFA6A65]:0)
FAILED: org.apache.solr.core.TestLazyCores.testLazySearch
Error Message:
Test abandoned because suite timeout was reached.
Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
at __randomizedtesting.SeedInfo.seed([16B355D9DBFA6A65]:0)
FAILED: org.apache.solr.core.TestLazyCores.testLazySearch
Error Message:
Test abandoned because suite timeout was reached.
Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
at __randomizedtesting.SeedInfo.seed([16B355D9DBFA6A65]:0)
Build Log:
[...truncated 1841 lines...]
[junit4] JVM J1: stdout was not empty, see: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/build/core/test/temp/junit4-J1-20180316_083419_2518528479762677445149.sysout
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] codec: FastDecompressionCompressingStoredFields, pf: LuceneFixedGap, dvf: Memory
[junit4] <<< JVM J1: EOF ----
[...truncated 11644 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.ScheduledTriggerTest
[junit4] 2> 4058877 INFO (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.ScheduledTriggerTest_16B355D9DBFA6A65-001/init-core-data-001
[junit4] 2> 4058877 INFO (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 4058878 INFO (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 4058879 INFO (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.ScheduledTriggerTest_16B355D9DBFA6A65-001/tempDir-001
[junit4] 2> 4058879 INFO (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 4058879 INFO (Thread-10832) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 4058879 INFO (Thread-10832) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 4058881 ERROR (Thread-10832) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 4058979 INFO (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:50783
[junit4] 2> 4058987 INFO (zkConnectionManagerCallback-11034-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4058993 INFO (jetty-launcher-11031-thread-1) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T00:27:37+03:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 4058994 INFO (jetty-launcher-11031-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 4058994 INFO (jetty-launcher-11031-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 4058994 INFO (jetty-launcher-11031-thread-1) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 4058994 INFO (jetty-launcher-11031-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@39bb82cd{/solr,null,AVAILABLE}
[junit4] 2> 4058994 INFO (jetty-launcher-11031-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1752b9f7{HTTP/1.1,[http/1.1]}{127.0.0.1:43318}
[junit4] 2> 4058994 INFO (jetty-launcher-11031-thread-1) [ ] o.e.j.s.Server Started @4063734ms
[junit4] 2> 4058995 INFO (jetty-launcher-11031-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43318}
[junit4] 2> 4058995 ERROR (jetty-launcher-11031-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 4058995 INFO (jetty-launcher-11031-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solrâ„¢ version 8.0.0
[junit4] 2> 4058995 INFO (jetty-launcher-11031-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 4058995 INFO (jetty-launcher-11031-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 4058995 INFO (jetty-launcher-11031-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-16T09:59:26.227Z
[junit4] 2> 4058997 INFO (zkConnectionManagerCallback-11036-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4058997 INFO (jetty-launcher-11031-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 4059004 INFO (jetty-launcher-11031-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50783/solr
[junit4] 2> 4059005 INFO (zkConnectionManagerCallback-11040-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4059008 INFO (zkConnectionManagerCallback-11042-thread-1-processing-n:127.0.0.1:43318_solr) [n:127.0.0.1:43318_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4059137 INFO (jetty-launcher-11031-thread-1) [n:127.0.0.1:43318_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 4059138 INFO (jetty-launcher-11031-thread-1) [n:127.0.0.1:43318_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43318_solr
[junit4] 2> 4059139 INFO (jetty-launcher-11031-thread-1) [n:127.0.0.1:43318_solr ] o.a.s.c.Overseer Overseer (id=72068813957758979-127.0.0.1:43318_solr-n_0000000000) starting
[junit4] 2> 4059149 INFO (zkConnectionManagerCallback-11047-thread-1-processing-n:127.0.0.1:43318_solr) [n:127.0.0.1:43318_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4059152 INFO (jetty-launcher-11031-thread-1) [n:127.0.0.1:43318_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50783/solr ready
[junit4] 2> 4059153 INFO (jetty-launcher-11031-thread-1) [n:127.0.0.1:43318_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43318_solr
[junit4] 2> 4059153 DEBUG (OverseerAutoScalingTriggerThread-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 4059154 DEBUG (OverseerAutoScalingTriggerThread-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 4059155 INFO (OverseerStateUpdate-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4059155 DEBUG (OverseerAutoScalingTriggerThread-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr ] o.a.s.c.a.NodeLostTrigger Initial livenodes: []
[junit4] 2> 4059155 DEBUG (OverseerAutoScalingTriggerThread-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
[junit4] 2> 4059155 DEBUG (OverseerAutoScalingTriggerThread-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 4059156 INFO (zkCallback-11046-thread-1-processing-n:127.0.0.1:43318_solr) [n:127.0.0.1:43318_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4059160 DEBUG (OverseerAutoScalingTriggerThread-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 4059161 DEBUG (OverseerAutoScalingTriggerThread-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 4059161 DEBUG (ScheduledTrigger-14251-thread-1) [n:127.0.0.1:43318_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 4059358 INFO (jetty-launcher-11031-thread-1) [n:127.0.0.1:43318_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43318.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14887272
[junit4] 2> 4059364 INFO (jetty-launcher-11031-thread-1) [n:127.0.0.1:43318_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43318.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14887272
[junit4] 2> 4059364 INFO (jetty-launcher-11031-thread-1) [n:127.0.0.1:43318_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43318.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@14887272
[junit4] 2> 4059365 INFO (jetty-launcher-11031-thread-1) [n:127.0.0.1:43318_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.ScheduledTriggerTest_16B355D9DBFA6A65-001/tempDir-001/node1/.
[junit4] 2> 4059380 INFO (zkConnectionManagerCallback-11049-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4059384 INFO (zkConnectionManagerCallback-11053-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 4059387 INFO (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4059388 INFO (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50783/solr ready
[junit4] 2> 4059486 INFO (TEST-ScheduledTriggerTest.testTrigger-seed#[16B355D9DBFA6A65]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testTrigger
[junit4] 2> 4060166 DEBUG (ScheduledTrigger-14251-thread-3) [n:127.0.0.1:43318_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 4061168 DEBUG (ScheduledTrigger-14251-thread-2) [n:127.0.0.1:43318_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 4062168 DEBUG (ScheduledTrigger-14251-thread-2) [n:127.0.0.1:43318_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 4062489 DEBUG (TEST-ScheduledTriggerTest.testTrigger-seed#[16B355D9DBFA6A65]) [ ] o.a.s.c.a.ScheduledTrigger ScheduledTrigger sched1 firing registered processor for scheduled time 2018-03-16T09:59:29.718Z, now=2018-03-16T09:59:32.289Z
[junit4] 2> 4063172 DEBUG (ScheduledTrigger-14251-thread-2) [n:127.0.0.1:43318_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 4064174 DEBUG (ScheduledTrigger-14251-thread-4) [n:127.0.0.1:43318_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 4065182 DEBUG (ScheduledTrigger-14251-thread-2) [n:127.0.0.1:43318_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 4065489 DEBUG (TEST-ScheduledTriggerTest.testTrigger-seed#[16B355D9DBFA6A65]) [ ] o.a.s.c.a.ScheduledTrigger ScheduledTrigger sched1 firing registered processor for scheduled time 2018-03-16T09:59:32.718Z, now=2018-03-16T09:59:35.290Z
[junit4] 2> 4066183 DEBUG (ScheduledTrigger-14251-thread-2) [n:127.0.0.1:43318_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 4067188 DEBUG (ScheduledTrigger-14251-thread-2) [n:127.0.0.1:43318_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 4068188 DEBUG (ScheduledTrigger-14251-thread-2) [n:127.0.0.1:43318_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 4068490 DEBUG (TEST-ScheduledTriggerTest.testTrigger-seed#[16B355D9DBFA6A65]) [ ] o.a.s.c.a.ScheduledTrigger ScheduledTrigger sched1 firing registered processor for scheduled time 2018-03-16T09:59:35.718Z, now=2018-03-16T09:59:38.291Z
[junit4] 2> 4068492 DEBUG (TEST-ScheduledTriggerTest.testTrigger-seed#[16B355D9DBFA6A65]) [ ] o.a.s.c.a.ScheduledTrigger ScheduledTrigger sched1 firing registered processor for scheduled time 2018-03-16T09:59:38Z, now=2018-03-16T09:59:38.293Z
[junit4] 2> 4068492 INFO (TEST-ScheduledTriggerTest.testTrigger-seed#[16B355D9DBFA6A65]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testTrigger
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ScheduledTriggerTest -Dtests.method=testTrigger -Dtests.seed=16B355D9DBFA6A65 -Dtests.slow=true -Dtests.locale=id-ID -Dtests.timezone=America/Lima -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 9.11s J1 | ScheduledTriggerTest.testTrigger <<<
[junit4] > Throwable #1: java.lang.AssertionError: Did not expect the listener to fire on first run!
[junit4] > at __randomizedtesting.SeedInfo.seed([16B355D9DBFA6A65:7578635B42351948]:0)
[junit4] > at org.apache.solr.cloud.autoscaling.ScheduledTriggerTest.lambda$new$0(ScheduledTriggerTest.java:48)
[junit4] > at org.apache.solr.cloud.autoscaling.ScheduledTrigger.run(ScheduledTrigger.java:191)
[junit4] > at org.apache.solr.cloud.autoscaling.ScheduledTriggerTest.scheduledTriggerTest(ScheduledTriggerTest.java:102)
[junit4] > at org.apache.solr.cloud.autoscaling.ScheduledTriggerTest.testTrigger(ScheduledTriggerTest.java:75)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 4068662 INFO (TEST-ScheduledTriggerTest.testIgnoredEvent-seed#[16B355D9DBFA6A65]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testIgnoredEvent
[junit4] 2> 4068663 WARN (TEST-ScheduledTriggerTest.testIgnoredEvent-seed#[16B355D9DBFA6A65]) [ ] o.a.s.c.a.ScheduledTrigger ScheduledTrigger was not able to run event at scheduled time: 2018-03-15T09:59:35.894Z. Now: 2018-03-16T09:59:38.465Z
[junit4] 2> 4068664 INFO (TEST-ScheduledTriggerTest.testIgnoredEvent-seed#[16B355D9DBFA6A65]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testIgnoredEvent
[junit4] 2> 4068667 INFO (jetty-closer-11032-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@1752b9f7{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 4068670 INFO (jetty-closer-11032-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1052262676
[junit4] 2> 4068671 INFO (jetty-closer-11032-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 4068671 INFO (jetty-closer-11032-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3c0a4450: rootName = solr_43318, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@612cb6f
[junit4] 2> 4068676 INFO (jetty-closer-11032-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 4068676 INFO (jetty-closer-11032-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@106784e1: rootName = solr_43318, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@686a409f
[junit4] 2> 4068681 INFO (jetty-closer-11032-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 4068681 INFO (jetty-closer-11032-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@438bb23c: rootName = solr_43318, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@6c55c94a
[junit4] 2> 4068683 INFO (jetty-closer-11032-thread-1) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:43318_solr
[junit4] 2> 4068684 INFO (jetty-closer-11032-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 4068684 INFO (jetty-closer-11032-thread-1) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 4068684 INFO (jetty-closer-11032-thread-1) [ ] o.a.s.c.Overseer Overseer (id=72068813957758979-127.0.0.1:43318_solr-n_0000000000) closing
[junit4] 2> 4068685 INFO (OverseerStateUpdate-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:43318_solr
[junit4] 2> 4068689 INFO (zkCallback-11041-thread-1-processing-n:127.0.0.1:43318_solr) [n:127.0.0.1:43318_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 4068687 WARN (OverseerAutoScalingTriggerThread-72068813957758979-127.0.0.1:43318_solr-n_0000000000) [n:127.0.0.1:43318_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 4068689 INFO (zkCallback-11046-thread-1-processing-n:127.0.0.1:43318_solr) [n:127.0.0.1:43318_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 4068689 DEBUG (jetty-closer-11032-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 4068696 INFO (jetty-closer-11032-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@39bb82cd{/solr,null,UNAVAILABLE}
[junit4] 2> 4068701 INFO (jetty-closer-11032-thread-1) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 4068703 ERROR (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 4068704 INFO (SUITE-ScheduledTriggerTest-seed#[16B355D9DBFA6A65]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:50783 50783
[junit4] 2> 4069607 INFO (Thread-10832) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:50783 50783
[junit4] 2> 4069620 WARN (Thread-10832) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/aliases.json
[junit4] 2> 3 /solr/clusterprops.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/live_nodes
[junit4] 2> 3 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.ScheduledTriggerTest_16B355D9DBFA6A65-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1341, maxMBSortInHeap=5.385787755099965, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@118fbc79), locale=id-ID, timezone=America/Lima
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_162 (64-bit)/cpus=3,threads=1,free=134922552,total=536870912
[junit4] 2> NOTE: All tests run in this JVM: [TestStressCloudBlindAtomicUpdates, TestRawResponseWriter, ZkControllerTest, SyncSliceTest, LoggingHandlerTest, TestPolicyCloud, SmileWriterTest, TestPayloadCheckQParserPlugin, AddBlockUpdateTest, DistributedFacetPivotSmallTest, ImplicitSnitchTest, TestCollectionAPI, TestLegacyNumericRangeQueryBuilder, TestAddFieldRealTimeGet, TestUniqueKeyFieldResource, SearchHandlerTest, UpdateRequestProcessorFactoryTest, CSVRequestHandlerTest, TestExtendedDismaxParser, ConnectionManagerTest, DistributedTermsComponentTest, TestMissingGroups, TestSolrCoreParser, ConcurrentCreateRoutedAliasTest, TestDistributedMissingSort, DistanceUnitsTest, TestTolerantUpdateProcessorCloud, TestFieldCache, NodeAddedTriggerTest, TestStressLucene, MinimalSchemaTest, TestCustomStream, AssignTest, TestFieldCacheVsDocValues, OverseerTest, TestStressReorder, TestEmbeddedSolrServerAdminHandler, HighlighterMaxOffsetTest, CdcrReplicationDistributedZkTest, DistributedDebugComponentTest, TestReplicaProperties, HdfsRecoverLeaseTest, URLClassifyProcessorTest, TestNumericTerms64, ClusterStateUpdateTest, TestManagedSchemaAPI, TestEmbeddedSolrServerSchemaAPI, TestFilteredDocIdSet, UpdateParamsTest, TestQuerySenderListener, TestRandomDVFaceting, AutoScalingHandlerTest, TriLevelCompositeIdRoutingTest, TestHashQParserPlugin, RankQueryTest, DirectoryFactoryTest, ParsingFieldUpdateProcessorsTest, TestSort, TestLRUCache, TestPHPSerializedResponseWriter, AtomicUpdateProcessorFactoryTest, TermVectorComponentTest, TestCollectionAPIs, TestFieldCacheSort, TestLegacyNumericUtils, TestValueSourceCache, ChaosMonkeyNothingIsSafeTest, DistribDocExpirationUpdateProcessorTest, TestDistribStateManager, TestFastLRUCache, AddReplicaTest, DistributedQueueTest, MetricsHandlerTest, TestSchemaResource, ChangedSchemaMergeTest, CollectionStateFormat2Test, ConnectionReuseTest, TestCloudPseudoReturnFields, DeleteLastCustomShardedReplicaTest, TestMultiValuedNumericRangeQuery, SoftAutoCommitTest, TestXmlQParserPlugin, TestJsonFacetsWithNestedObjects, TestNoOpRegenerator, SpellCheckCollatorWithCollapseTest, TestComplexPhraseLeadingWildcard, TokenizerChainTest, SuggestComponentContextFilterQueryTest, TestCSVLoader, CoreAdminHandlerTest, CdcrRequestHandlerTest, LeaderVoteWaitTimeoutTest, TestRestManager, BlockJoinFacetRandomTest, TestInfoStreamLogging, ClassificationUpdateProcessorTest, OverseerModifyCollectionTest, AnalyticsMergeStrategyTest, RestartWhileUpdatingTest, TestTestInjection, ScriptEngineTest, DistributedSpellCheckComponentTest, FieldAnalysisRequestHandlerTest, TestGenericDistributedQueue, ZkSolrClientTest, CdcrUpdateLogTest, TestSearcherReuse, RemoteQueryErrorTest, TestIndexingPerformance, DistributedSuggestComponentTest, TestHdfsCloudBackupRestore, TestConfigReload, TimeZoneUtilsTest, HdfsLockFactoryTest, SpellCheckCollatorTest, TestSchemalessBufferedUpdates, TestRandomFlRTGCloud, EchoParamsTest, DistributedQueryComponentCustomSortTest, AssignBackwardCompatibilityTest, CdcrBidirectionalTest, HdfsUnloadDistributedZkTest, TestAtomicUpdateErrorCases, DefaultValueUpdateProcessorTest, ClusterStateTest, TestSolrQueryParser, StressHdfsTest, ExternalFileFieldSortTest, TestNRTOpen, TestCoreDiscovery, DeleteReplicaTest, TestConfigSetsAPIZkFailure, TestSchemaVersionResource, BlockCacheTest, ExecutePlanActionTest, TestCloudSchemaless, ShardRoutingTest, TestRTGBase, TestQueryWrapperFilter, TestMaxTokenLenTokenizer, DOMUtilTest, InfixSuggestersTest, PeerSyncReplicationTest, TestLeaderElectionZkExpiry, TestConfigSets, HDFSCollectionsAPITest, SystemInfoHandlerTest, DisMaxRequestHandlerTest, TestDFRSimilarityFactory, TestRetrieveFieldsOptimizer, TestConfigSetImmutable, SolrCmdDistributorTest, TestLegacyField, EnumFieldTest, TestStressRecovery, TestNonDefinedSimilarityFactory, DistributedIntervalFacetingTest, SliceStateTest, TestStressLiveNodes, UUIDFieldTest, TestBulkSchemaConcurrent, BasicFunctionalityTest, TestRandomFaceting, TestWordDelimiterFilterFactory, ScheduledMaintenanceTriggerTest, MoveReplicaTest, TestDocSet, BadCopyFieldTest, HdfsThreadLeakTest, SolrCoreCheckLockOnStartupTest, NoCacheHeaderTest, LeaderFailoverAfterPartitionTest, SharedFSAutoReplicaFailoverTest, VersionInfoTest, V2StandaloneTest, LIRRollingUpdatesTest, TestInPlaceUpdatesStandalone, TestNumericRangeQuery64, TestCodecSupport, ExplicitHLLTest, TestLegacyFieldCache, TestStressVersions, TestCloudDeleteByQuery, HdfsDirectoryFactoryTest, TestStandardQParsers, TestSmileRequest, TestExclusionRuleCollectionAccess, PKIAuthenticationIntegrationTest, TestReqParamsAPI, ResponseLogComponentTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, OpenCloseCoreStressTest, LeaderElectionIntegrationTest, BasicZkTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, LeaderElectionTest, TestZkChroot, ZkCLITest, TestDistributedSearch, TestDistributedGrouping, TestFaceting, TestRecovery, TestHashPartitioner, TestRangeQuery, TestCoreContainer, SimpleFacetsTest, TestSolr4Spatial, StatsComponentTest, SpellCheckComponentTest, TestGroupingSearch, QueryElevationComponentTest, PeerSyncTest, BadIndexSchemaTest, TestFiltering, TestFunctionQuery, DirectUpdateHandlerTest, TestBadConfig, HighlighterTest, ShowFileRequestHandlerTest, SolrIndexSplitterTest, SimplePostToolTest, SignatureUpdateProcessorFactoryTest, SuggesterFSTTest, SolrRequestParserTest, TestFoldingMultitermQuery, DocValuesTest, SuggesterTSTTest, SuggesterTest, SpatialFilterTest, SuggesterWFSTTest, PolyFieldTest, WordBreakSolrSpellCheckerTest, SchemaVersionSpecificBehaviorTest, TestPseudoReturnFields, TestUpdate, FieldMutatingUpdateProcessorTest, QueryEqualityTest, StatelessScriptUpdateProcessorFactoryTest, SortByFunctionTest, TestRemoteStreaming, DocValuesMultiTest, DistanceFunctionTest, TestSolrDeletionPolicy1, SolrInfoBeanTest, XsltUpdateRequestHandlerTest, DebugComponentTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, TestQueryUtils, TestWriterPerf, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, DocumentAnalysisRequestHandlerTest, TestQueryTypes, PrimitiveFieldTypeTest, FileBasedSpellCheckerTest, RequestHandlersTest, DocumentBuilderTest, PathHierarchyTokenizerFactoryTest, RequiredFieldsTest, FastVectorHighlighterTest, RegexBoostProcessorTest, SolrPluginUtilsTest, IndexSchemaTest, TestCollationField, ReturnFieldsTest, TestCSVResponseWriter, MBeansHandlerTest, QueryParsingTest, TestAnalyzedSuggestions, JsonLoaderTest, TestPartialUpdateDeduplication, PingRequestHandlerTest, TestComponentsName, SearchHandlerTest, TestLFUCache, HighlighterConfigTest, TestSolrIndexConfig, BadComponentTest, TestSolrDeletionPolicy2, SampleTest, TestSearchPerf, NumericFieldsTest, TestConfig, TestFuzzyAnalyzedSuggestions, TestSolrCoreProperties, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestLuceneMatchVersion, SpellPossibilityIteratorTest, TestCharFilters, TestXIncludeConfig, TestSweetSpotSimilarityFactory, TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestIBSimilarityFactory, TestBM25SimilarityFactory, TestFastWriter, ResourceLoaderTest, OpenExchangeRatesOrgProviderTest, PluginInfoTest, DateMathParserTest, PreAnalyzedFieldTest, PrimUtilsTest, TestSuggestSpellingConverter, DateFieldTest, TestDocumentBuilder, CircularListTest, CursorPagingTest, TestCrossCoreJoin, TestSimpleTrackingShardHandler, ActionThrottleTest, AliasIntegrationTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, ChaosMonkeySafeLeaderWithPullReplicasTest, CloudExitableDirectoryReaderTest, CollectionPropsTest, CollectionsAPISolrJTest, CreateRoutedAliasTest, DeleteInactiveReplicaTest, DeleteShardTest, DeleteStatusTest, DistributedVersionInfoTest, DocValuesNotIndexedTest, HealthCheckHandlerTest, HttpPartitionTest, LIROnShardRestartTest, LeaderFailureAfterFreshStartTest, LegacyCloudClusterPropTest, MigrateRouteKeyTest, MissingSegmentRecoveryTest, MoveReplicaHDFSFailoverTest, MoveReplicaHDFSTest, NodeMutatorTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerStatusTest, OverseerTaskQueueTest, ReplaceNodeNoTargetTest, ReplaceNodeTest, SSLMigrationTest, SaslZkACLProviderTest, SolrCLIZkUtilsTest, SolrCloudExampleTest, SolrXmlInZkTest, TestAuthenticationFramework, TestCloudConsistency, TestCloudJSONFacetJoinDomain, TestCloudPivotFacet, TestCloudRecovery, TestCloudSearcherWarming, TestClusterProperties, TestConfigSetsAPI, TestDeleteCollectionOnDownNodes, TestDistribDocBasedVersion, TestDistributedMap, TestLeaderInitiatedRecoveryThread, TestLockTree, TestOnReconnectListenerSupport, TestPullReplica, TestRebalanceLeaders, TestRequestForwarding, TestSSLRandomization, TestShortCircuitedRequests, TestSkipOverseerOperations, TestSolrCloudWithKerberosAlt, TestUtilizeNode, VMParamsZkACLAndCredentialsProvidersTest, ZkFailoverTest, ZkShardTermsTest, CollectionTooManyReplicasTest, CollectionsAPIAsyncDistributedZkTest, CollectionsAPIDistributedZkTest, CustomCollectionTest, HdfsCollectionsAPIDistributedZkTest, ShardSplitTest, TestCollectionsAPIViaSolrCloudCluster, AutoAddReplicasIntegrationTest, AutoAddReplicasPlanActionTest, ComputePlanActionTest, HdfsAutoAddReplicasIntegrationTest, HttpTriggerListenerTest, MetricTriggerTest, NodeLostTriggerTest, ScheduledTriggerTest]
[junit4] Completed [398/785 (1!)] on J1 in 10.79s, 2 tests, 1 failure <<< FAILURES!
[...truncated 692 lines...]
[junit4] Suite: org.apache.solr.core.TestLazyCores
[junit4] 2> 11257 INFO (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/init-core-data-001
[junit4] 2> 11259 WARN (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=10 numCloses=10
[junit4] 2> 11260 INFO (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 11262 INFO (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 11264 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testCreateSame
[junit4] 2> 11286 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/solr.xml
[junit4] 2> 11291 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 11292 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 11374 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 11458 INFO (coreLoadExecutor-72-thread-3) [ x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 11473 INFO (coreLoadExecutor-72-thread-2) [ x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 11478 INFO (coreLoadExecutor-72-thread-3) [ x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
[junit4] 2> 11485 INFO (coreLoadExecutor-72-thread-3) [ x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 11485 INFO (coreLoadExecutor-72-thread-3) [ x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection5, trusted=true
[junit4] 2> 11494 INFO (coreLoadExecutor-72-thread-3) [ x:collection5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 11494 INFO (coreLoadExecutor-72-thread-3) [ x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection5/data/]
[junit4] 2> 11497 INFO (coreLoadExecutor-72-thread-3) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 11498 INFO (coreLoadExecutor-72-thread-2) [ x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
[junit4] 2> 11504 INFO (coreLoadExecutor-72-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 11513 INFO (coreLoadExecutor-72-thread-2) [ x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 11515 INFO (coreLoadExecutor-72-thread-2) [ x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection2, trusted=true
[junit4] 2> 11515 INFO (coreLoadExecutor-72-thread-2) [ x:collection2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 11515 INFO (coreLoadExecutor-72-thread-2) [ x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection2/data/]
[junit4] 2> 11520 INFO (coreLoadExecutor-72-thread-2) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 11526 INFO (coreLoadExecutor-72-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
[junit4] 2> 11529 INFO (coreLoadExecutor-72-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 11529 INFO (coreLoadExecutor-72-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection1, trusted=true
[junit4] 2> 11530 INFO (coreLoadExecutor-72-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 11530 INFO (coreLoadExecutor-72-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection1/data/]
[junit4] 2> 11532 INFO (coreLoadExecutor-72-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 11722 INFO (coreLoadExecutor-72-thread-2) [ x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 11722 INFO (coreLoadExecutor-72-thread-2) [ x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 11723 INFO (coreLoadExecutor-72-thread-2) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 11751 INFO (coreLoadExecutor-72-thread-2) [ x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@4b4aef1b[collection2] main]
[junit4] 2> 11752 INFO (coreLoadExecutor-72-thread-2) [ x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection2/conf
[junit4] 2> 11753 INFO (coreLoadExecutor-72-thread-2) [ x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 11759 INFO (searcherExecutor-74-thread-1-processing-x:collection2) [ x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@4b4aef1b[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 11775 INFO (coreLoadExecutor-72-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 11775 INFO (coreLoadExecutor-72-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 11776 INFO (coreLoadExecutor-72-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 11778 INFO (coreLoadExecutor-72-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@415e3684[collection1] main]
[junit4] 2> 11778 INFO (coreLoadExecutor-72-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection1/conf
[junit4] 2> 11779 INFO (coreLoadExecutor-72-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 11782 INFO (searcherExecutor-75-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@415e3684[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 11793 INFO (coreLoadExecutor-72-thread-3) [ x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 11793 INFO (coreLoadExecutor-72-thread-3) [ x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 11794 INFO (coreLoadExecutor-72-thread-3) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 11796 INFO (coreLoadExecutor-72-thread-3) [ x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@57d7f404[collection5] main]
[junit4] 2> 11799 INFO (coreLoadExecutor-72-thread-3) [ x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection5/conf
[junit4] 2> 11800 INFO (coreLoadExecutor-72-thread-3) [ x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 11802 INFO (searcherExecutor-73-thread-1-processing-x:collection5) [ x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@57d7f404[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 11820 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 11837 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.s.IndexSchema [collection4] Schema name=tiny
[junit4] 2> 11841 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 11841 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'collection4' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection4, trusted=true
[junit4] 2> 11841 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 11841 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrCore [[collection4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection4/data/]
[junit4] 2> 11846 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=45, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 11945 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 11945 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 11946 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=47, maxMergedSegmentMB=45.869140625, floorSegmentMB=1.318359375, forceMergeDeletesPctAllowed=18.995916436538693, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 11947 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@3604b76b[collection4] main]
[junit4] 2> 11948 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection4/conf
[junit4] 2> 11948 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 11950 INFO (searcherExecutor-86-thread-1-processing-x:collection4) [ x:collection4] o.a.s.c.SolrCore [collection4] Registered new searcher Searcher@3604b76b[collection4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 11968 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 11981 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.s.IndexSchema [collection6] Schema name=tiny
[junit4] 2> 11986 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 11986 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.CoreContainer Creating SolrCore 'collection6' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection6, trusted=true
[junit4] 2> 11986 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 11986 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrCore [[collection6] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection6], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/./collection6/data/]
[junit4] 2> 11989 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 12133 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 12133 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 12135 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1014728150, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 12140 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.s.SolrIndexSearcher Opening [Searcher@19cfe4c8[collection6] main]
[junit4] 2> 12141 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-001/collection6/conf
[junit4] 2> 12141 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 12159 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.h.a.CoreAdminOperation core create command schema=schema.xml&dataDir=t12&name=collection2&action=CREATE&config=solrconfig.xml&wt=xml
[junit4] 2> 12159 WARN (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.CoreContainer Creating a core with existing name is not allowed
[junit4] 2> 12160 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.h.a.CoreAdminOperation core create command schema=schema.xml&dataDir=t14&name=collection4&action=CREATE&config=solrconfig.xml&wt=xml
[junit4] 2> 12160 WARN (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.CoreContainer Creating a core with existing name is not allowed
[junit4] 2> 12160 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.h.a.CoreAdminOperation core create command schema=schema.xml&dataDir=t15&name=collection5&action=CREATE&config=solrconfig.xml&wt=xml
[junit4] 2> 12160 WARN (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.CoreContainer Creating a core with existing name is not allowed
[junit4] 2> 12160 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.h.a.CoreAdminOperation core create command schema=schema.xml&dataDir=t16&name=collection6&action=CREATE&config=solrconfig.xml&wt=xml
[junit4] 2> 12188 WARN (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.CoreContainer Creating a core with existing name is not allowed
[junit4] 2> 12188 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.CoreContainer Shutting down CoreContainer instance=511387418
[junit4] 2> 12188 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 12188 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 12189 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 12189 INFO (searcherExecutor-90-thread-1-processing-x:collection4) [ x:collection4] o.a.s.c.SolrCore [collection6] Registered new searcher Searcher@19cfe4c8[collection6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 12191 INFO (coreCloseExecutor-94-thread-1-processing-x:collection4) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@fa0a3d8
[junit4] 2> 12191 INFO (coreCloseExecutor-94-thread-1-processing-x:collection4) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=262185944
[junit4] 2> 12209 INFO (coreCloseExecutor-94-thread-4-processing-x:collection4) [ x:collection2] o.a.s.c.SolrCore [collection2] CLOSING SolrCore org.apache.solr.core.SolrCore@2ff4053c
[junit4] 2> 12209 INFO (coreCloseExecutor-94-thread-4-processing-x:collection4) [ x:collection2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection2, tag=804521276
[junit4] 2> 12210 INFO (coreCloseExecutor-94-thread-2-processing-x:collection4) [ x:collection5] o.a.s.c.SolrCore [collection5] CLOSING SolrCore org.apache.solr.core.SolrCore@113d3f5c
[junit4] 2> 12210 INFO (coreCloseExecutor-94-thread-2-processing-x:collection4) [ x:collection5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection5, tag=289226588
[junit4] 2> 12212 INFO (coreCloseExecutor-94-thread-3-processing-x:collection4) [ x:collection4] o.a.s.c.SolrCore [collection4] CLOSING SolrCore org.apache.solr.core.SolrCore@69aa006b
[junit4] 2> 12212 INFO (coreCloseExecutor-94-thread-3-processing-x:collection4) [ x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection4, tag=1772748907
[junit4] 2> 12212 INFO (coreCloseExecutor-94-thread-5-processing-x:collection4) [ x:collection6] o.a.s.c.SolrCore [collection6] CLOSING SolrCore org.apache.solr.core.SolrCore@794bf524
[junit4] 2> 12212 INFO (coreCloseExecutor-94-thread-5-processing-x:collection4) [ x:collection6] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection6, tag=2035021092
[junit4] 2> 12226 INFO (TEST-TestLazyCores.testCreateSame-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.SolrTestCaseJ4 ###Ending testCreateSame
[junit4] 2> 12229 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.SolrTestCaseJ4 ###Starting testMidUseUnload
[junit4] 2> 12253 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/solr.xml
[junit4] 2> 12258 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 12258 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 12301 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 12333 INFO (coreLoadExecutor-101-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 12393 INFO (coreLoadExecutor-101-thread-1-processing-x:collection4) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 12402 INFO (coreLoadExecutor-101-thread-1-processing-x:collection4) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
[junit4] 2> 12414 INFO (coreLoadExecutor-101-thread-2-processing-x:collection4) [ x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
[junit4] 2> 12417 INFO (coreLoadExecutor-101-thread-2-processing-x:collection4) [ x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 12417 INFO (coreLoadExecutor-101-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/./collection2, trusted=true
[junit4] 2> 12417 INFO (coreLoadExecutor-101-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 12417 INFO (coreLoadExecutor-101-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/./collection2/data/]
[junit4] 2> 12420 INFO (coreLoadExecutor-101-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 12422 INFO (coreLoadExecutor-101-thread-1-processing-x:collection4) [ x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 12422 INFO (coreLoadExecutor-101-thread-1-processing-x:collection4) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/./collection1, trusted=true
[junit4] 2> 12422 INFO (coreLoadExecutor-101-thread-1-processing-x:collection4) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 12422 INFO (coreLoadExecutor-101-thread-1-processing-x:collection4) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/./collection1/data/]
[junit4] 2> 12425 INFO (coreLoadExecutor-101-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 12432 INFO (coreLoadExecutor-101-thread-3-processing-x:collection4) [ x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 12490 INFO (coreLoadExecutor-101-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 12490 INFO (coreLoadExecutor-101-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 12491 INFO (coreLoadExecutor-101-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 12536 INFO (coreLoadExecutor-101-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 12536 INFO (coreLoadExecutor-101-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 12537 INFO (coreLoadExecutor-101-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 12546 INFO (coreLoadExecutor-101-thread-3-processing-x:collection4) [ x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
[junit4] 2> 12550 INFO (coreLoadExecutor-101-thread-3-processing-x:collection4) [ x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 12550 INFO (coreLoadExecutor-101-thread-3-processing-x:collection4) [ x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/./collection5, trusted=true
[junit4] 2> 12550 INFO (coreLoadExecutor-101-thread-3-processing-x:collection4) [ x:collection5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 12550 INFO (coreLoadExecutor-101-thread-3-processing-x:collection4) [ x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/./collection5/data/]
[junit4] 2> 12552 INFO (coreLoadExecutor-101-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 12554 INFO (coreLoadExecutor-101-thread-1-processing-x:collection4) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@85e1127[collection1] main]
[junit4] 2> 12554 INFO (coreLoadExecutor-101-thread-1-processing-x:collection4) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/collection1/conf
[junit4] 2> 12555 INFO (coreLoadExecutor-101-thread-1-processing-x:collection4) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 12560 INFO (searcherExecutor-103-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@85e1127[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 12567 INFO (coreLoadExecutor-101-thread-2-processing-x:collection4) [ x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@70be7fd3[collection2] main]
[junit4] 2> 12568 INFO (coreLoadExecutor-101-thread-2-processing-x:collection4) [ x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/collection2/conf
[junit4] 2> 12569 INFO (coreLoadExecutor-101-thread-2-processing-x:collection4) [ x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 12570 INFO (searcherExecutor-102-thread-1-processing-x:collection2) [ x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@70be7fd3[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 12629 INFO (coreLoadExecutor-101-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 12629 INFO (coreLoadExecutor-101-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 12630 INFO (coreLoadExecutor-101-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 12631 INFO (coreLoadExecutor-101-thread-3-processing-x:collection4) [ x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@5fdf2bcf[collection5] main]
[junit4] 2> 12631 INFO (coreLoadExecutor-101-thread-3-processing-x:collection4) [ x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-002/collection5/conf
[junit4] 2> 12632 INFO (coreLoadExecutor-101-thread-3-processing-x:collection4) [ x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 12633 INFO (searcherExecutor-110-thread-1-processing-x:collection5) [ x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@5fdf2bcf[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 12638 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=null
[junit4] 2> 12639 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrCore Core collection1 is not yet closed, waiting 100 ms before checking again.
[junit4] 2> 12739 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrCore Core collection1 is not yet closed, waiting 100 ms before checking again.
[junit4] 2> 12839 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrCore Core collection1 is not yet closed, waiting 100 ms before checking again.
[junit4] 2> 12942 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrCore Core collection1 is not yet closed, waiting 100 ms before checking again.
[junit4] 2> 13043 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrCore Core collection1 is not yet closed, waiting 100 ms before checking again.
[junit4] 2> 13147 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrCore Core collection1 is not yet closed, waiting 100 ms before checking again.
[junit4] 2> 13248 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrCore Core collection1 is not yet closed, waiting 100 ms before checking again.
[junit4] 2> 13348 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrCore Core collection1 is not yet closed, waiting 100 ms before checking again.
[junit4] 2> 13438 INFO (Thread-58) [ x:collection4] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@40d0716f
[junit4] 2> 13438 INFO (Thread-58) [ x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1087402351
[junit4] 2> 13448 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1613191220
[junit4] 2> 13448 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 13448 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 13448 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 13460 INFO (coreCloseExecutor-115-thread-1-processing-x:collection4) [ x:collection5] o.a.s.c.SolrCore [collection5] CLOSING SolrCore org.apache.solr.core.SolrCore@265e60f0
[junit4] 2> 13460 INFO (coreCloseExecutor-115-thread-1-processing-x:collection4) [ x:collection5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection5, tag=643719408
[junit4] 2> 13460 INFO (coreCloseExecutor-115-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.SolrCore [collection2] CLOSING SolrCore org.apache.solr.core.SolrCore@6d5ce0b8
[junit4] 2> 13460 INFO (coreCloseExecutor-115-thread-2-processing-x:collection4) [ x:collection2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection2, tag=1834803384
[junit4] 2> 13466 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.SolrTestCaseJ4 ###Ending testMidUseUnload
[junit4] 2> 13489 INFO (TEST-TestLazyCores.testRace-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.SolrTestCaseJ4 ###Starting testRace
[junit4] 2> 13507 INFO (TEST-TestLazyCores.testRace-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/solr.xml
[junit4] 2> 13512 INFO (TEST-TestLazyCores.testRace-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 13512 INFO (TEST-TestLazyCores.testRace-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 13577 INFO (TEST-TestLazyCores.testRace-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 13610 INFO (coreLoadExecutor-122-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 13622 INFO (coreLoadExecutor-122-thread-2-processing-x:collection4) [ x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
[junit4] 2> 13625 INFO (coreLoadExecutor-122-thread-2-processing-x:collection4) [ x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 13625 INFO (coreLoadExecutor-122-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/./collection2, trusted=true
[junit4] 2> 13626 INFO (coreLoadExecutor-122-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 13626 INFO (coreLoadExecutor-122-thread-2-processing-x:collection4) [ x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/./collection2/data/]
[junit4] 2> 13614 INFO (coreLoadExecutor-122-thread-1-processing-x:collection4) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 13639 INFO (coreLoadExecutor-122-thread-1-processing-x:collection4) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
[junit4] 2> 13641 INFO (coreLoadExecutor-122-thread-1-processing-x:collection4) [ x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 13641 INFO (coreLoadExecutor-122-thread-1-processing-x:collection4) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/./collection1, trusted=true
[junit4] 2> 13642 INFO (coreLoadExecutor-122-thread-1-processing-x:collection4) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 13642 INFO (coreLoadExecutor-122-thread-1-processing-x:collection4) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/./collection1/data/]
[junit4] 2> 13645 INFO (coreLoadExecutor-122-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 13646 INFO (coreLoadExecutor-122-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 13651 INFO (coreLoadExecutor-122-thread-3-processing-x:collection4) [ x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 13668 INFO (coreLoadExecutor-122-thread-3-processing-x:collection4) [ x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
[junit4] 2> 13670 INFO (coreLoadExecutor-122-thread-3-processing-x:collection4) [ x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 13670 INFO (coreLoadExecutor-122-thread-3-processing-x:collection4) [ x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/./collection5, trusted=true
[junit4] 2> 13672 INFO (coreLoadExecutor-122-thread-3-processing-x:collection4) [ x:collection5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 13672 INFO (coreLoadExecutor-122-thread-3-processing-x:collection4) [ x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/./collection5/data/]
[junit4] 2> 13677 INFO (coreLoadExecutor-122-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 14038 INFO (coreLoadExecutor-122-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 14038 INFO (coreLoadExecutor-122-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 14038 INFO (coreLoadExecutor-122-thread-1-processing-x:collection4) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 14053 INFO (coreLoadExecutor-122-thread-1-processing-x:collection4) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@56e5c311[collection1] main]
[junit4] 2> 14129 INFO (coreLoadExecutor-122-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 14129 INFO (coreLoadExecutor-122-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 14130 INFO (coreLoadExecutor-122-thread-2-processing-x:collection4) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 14166 INFO (coreLoadExecutor-122-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 14166 INFO (coreLoadExecutor-122-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 14166 INFO (coreLoadExecutor-122-thread-3-processing-x:collection4) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 14167 INFO (coreLoadExecutor-122-thread-1-processing-x:collection4) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/collection1/conf
[junit4] 2> 14167 INFO (coreLoadExecutor-122-thread-1-processing-x:collection4) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 14168 INFO (coreLoadExecutor-122-thread-2-processing-x:collection4) [ x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@11807a5b[collection2] main]
[junit4] 2> 14169 INFO (coreLoadExecutor-122-thread-2-processing-x:collection4) [ x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/collection2/conf
[junit4] 2> 14169 INFO (coreLoadExecutor-122-thread-2-processing-x:collection4) [ x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 14170 INFO (coreLoadExecutor-122-thread-3-processing-x:collection4) [ x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@5739a46e[collection5] main]
[junit4] 2> 14171 INFO (coreLoadExecutor-122-thread-3-processing-x:collection4) [ x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/collection5/conf
[junit4] 2> 14171 INFO (coreLoadExecutor-122-thread-3-processing-x:collection4) [ x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 14194 INFO (searcherExecutor-124-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@56e5c311[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 14195 INFO (searcherExecutor-123-thread-1-processing-x:collection2) [ x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@11807a5b[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 14216 INFO (Thread-74) [ x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 14251 INFO (searcherExecutor-125-thread-1-processing-x:collection5) [ x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@5739a46e[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 14256 INFO (Thread-74) [ x:collection3] o.a.s.s.IndexSchema [collection3] Schema name=tiny
[junit4] 2> 14286 INFO (Thread-74) [ x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 14286 INFO (Thread-74) [ x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection3' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/./collection3, trusted=true
[junit4] 2> 14286 INFO (Thread-74) [ x:collection3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 14286 INFO (Thread-74) [ x:collection3] o.a.s.c.SolrCore [[collection3] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/collection3], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001/tempDir-003/./collection3/data/]
[junit4] 2> 14299 INFO (Thread-74) [ x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=24, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 14408 INFO (Thread-74) [ x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 14408 INFO (Thread-74) [ x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 14408 INFO (Thread-74) [ x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 14410 INFO (Thread-74) [
[...truncated too long message...]
o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@59ef38aa[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 7304286 INFO (TEST-TestLazyCores.testLazySearch-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1177801611
[junit4] 2> 7304286 WARN (TEST-TestLazyCores.testLazySearch-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.m.SolrMetricManager Interrupted while trying to obtain lock to modify reporters registry: solr.node
[junit4] 2> 7304286 INFO (TEST-TestLazyCores.testLazySearch-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 7304286 INFO (TEST-TestLazyCores.testLazySearch-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@748ccd53: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@670997af
[junit4] 2> 7304288 INFO (TEST-TestLazyCores.testLazySearch-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 7304288 INFO (TEST-TestLazyCores.testLazySearch-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@525466f: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@69f1cfd3
[junit4] 2> 7304289 INFO (coreCloseExecutor-192-thread-1-processing-x:collection4) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@7d090299
[junit4] 2> 7304289 INFO (coreCloseExecutor-192-thread-1-processing-x:collection4) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=2097742489
[junit4] 2> 7304289 INFO (coreCloseExecutor-192-thread-1-processing-x:collection4) [ x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@56a1385e: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@71a9294e
[junit4] 2> 7304289 INFO (coreCloseExecutor-192-thread-2-processing-x:collection4) [ x:collection5] o.a.s.c.SolrCore [collection5] CLOSING SolrCore org.apache.solr.core.SolrCore@fbbed29
[junit4] 2> 7304289 INFO (coreCloseExecutor-192-thread-3-processing-x:collection4) [ x:collection2] o.a.s.c.SolrCore [collection2] CLOSING SolrCore org.apache.solr.core.SolrCore@776e593b
[junit4] 2> 7304294 INFO (coreCloseExecutor-192-thread-2-processing-x:collection4) [ x:collection5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection5, tag=263974185
[junit4] 2> 7304294 INFO (coreCloseExecutor-192-thread-2-processing-x:collection4) [ x:collection5] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@17fe3705: rootName = null, domain = solr.core.collection5, service url = null, agent id = null] for registry solr.core.collection5 / com.codahale.metrics.MetricRegistry@61470c75
[junit4] 2> 7304302 INFO (coreCloseExecutor-192-thread-3-processing-x:collection4) [ x:collection2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection2, tag=2003720507
[junit4] 2> 7304302 INFO (coreCloseExecutor-192-thread-3-processing-x:collection4) [ x:collection2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@69ef3bd4: rootName = null, domain = solr.core.collection2, service url = null, agent id = null] for registry solr.core.collection2 / com.codahale.metrics.MetricRegistry@7fb25968
[junit4] 2> 7304311 INFO (TEST-TestLazyCores.testLazySearch-seed#[16B355D9DBFA6A65]) [ x:collection4] o.a.s.SolrTestCaseJ4 ###Ending testLazySearch
[junit4] 2> 7304312 INFO (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [ x:collection4] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> 7304312 INFO (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [ x:collection4] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1177801611
[junit4] 2> 7304312 INFO (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [ x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 7304312 INFO (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [ x:collection4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5247cc41: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@6b537520
[junit4] 2> 7304315 INFO (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [ x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 7304315 INFO (SUITE-TestLazyCores-seed#[16B355D9DBFA6A65]-worker) [ x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_16B355D9DBFA6A65-001
[junit4] 2> Mar 16, 2018 1:13:11 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> SEVERE: There are still zombie threads that couldn't be terminated:
[junit4] 2> 1) Thread[id=191, name=searcherExecutor-136-thread-1, state=WAITING, group=TGRP-TestLazyCores]
[junit4] 2> at sun.misc.Unsafe.park(Native Method)
[junit4] 2> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] 2> at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {v_t=BlockTreeOrds(blocksize=128), id=BlockTreeOrds(blocksize=128), v_s=Lucene50(blocksize=128), v_f=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{}, maxPointsInLeafNode=179, maxMBSortInHeap=6.340553083811743, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@6a1f220c), locale=id-ID, timezone=America/Lima
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_162 (64-bit)/cpus=3,threads=2,free=174467576,total=359661568
[junit4] 2> NOTE: All tests run in this JVM: [ScheduledTriggerTest, TestLazyCores]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestLazyCores -Dtests.seed=16B355D9DBFA6A65 -Dtests.slow=true -Dtests.badapples=true -Dtests.locale=id-ID -Dtests.timezone=America/Lima -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J1 | TestLazyCores (suite) <<<
[junit4] > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
[junit4] > at __randomizedtesting.SeedInfo.seed([16B355D9DBFA6A65]:0)
[junit4] Completed [8/10 (1!)] on J1 in 7283.30s, 6 tests, 2 errors <<< FAILURES!
[...truncated 17 lines...]
BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/common-build.xml:1561: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/common-build.xml:1085: There were test failures: 10 suites, 52 tests, 1 suite-level error, 1 error, 4 ignored [seed: 16B355D9DBFA6A65]
Total time: 121 minutes 54 seconds
[repro] Setting last failure code to 256
[repro] Failures:
[repro] 0/5 failed: org.apache.solr.cloud.autoscaling.ScheduledTriggerTest
[repro] 1/5 failed: org.apache.solr.core.TestLazyCores
[repro] Exiting with code 256
+ mv lucene/build lucene/build.repro
+ mv solr/build solr/build.repro
+ mv lucene/build.orig lucene/build
+ mv solr/build.orig solr/build
Archiving artifacts
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
[WARNINGS] Parsing warnings in console log with parser Java Compiler (javac)
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
<Git Blamer> Using GitBlamer to create author and commit information for all warnings.
<Git Blamer> GIT_COMMIT=2ca741d36a3078e7d7b03cb73176a1e99377eefc, workspace=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris
[WARNINGS] Computing warning deltas based on reference build #1735
Recording test results
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/export/home/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build #
1735 - Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1735/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC
3 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest
Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:41409 within 30000 ms
Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:41409 within 30000 ms
at __randomizedtesting.SeedInfo.seed([DF44788DBE20EF01]:0)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:183)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:120)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:115)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:102)
at org.apache.solr.cloud.MiniSolrCloudCluster.waitForAllNodes(MiniSolrCloudCluster.java:268)
at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:262)
at org.apache.solr.cloud.SolrCloudTestCase$Builder.configure(SolrCloudTestCase.java:190)
at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.setupCluster(DeleteLastCustomShardedReplicaTest.java:31)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:41409 within 30000 ms
at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:232)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:175)
... 31 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest
Error Message:
28 threads leaked from SUITE scope at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest: 1) Thread[id=7335, name=qtp1329013312-7335-acceptor-0@ac94e40-ServerConnector@247acfac{HTTP/1.1,[http/1.1]}{127.0.0.1:58786}, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=7363, name=qtp1661555970-7363, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=7327, name=ProcessThread(sid:0 cport:41409):, state=WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:123) 4) Thread[id=7347, name=qtp1661555970-7347, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=7325, name=SessionTracker, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at java.lang.Object.wait(Native Method) at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:147) 6) Thread[id=7355, name=qtp1329013312-7355, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=7324, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:196) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=7348, name=qtp1661555970-7348, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=7334, name=qtp1329013312-7334, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=7360, name=qtp1661555970-7360, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=7364, name=qtp1661555970-7364, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=7350, name=qtp1661555970-7350-acceptor-0@1bfd09c6-ServerConnector@3c4c0ab3{HTTP/1.1,[http/1.1]}{127.0.0.1:41409}, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=7361, name=qtp1661555970-7361, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=7333, name=qtp1329013312-7333, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.io.ManagedSelector$$Lambda$28/664856532.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=7362, name=Scheduler-1842259731, state=WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=7323, name=Thread-2090, state=WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at java.lang.Object.wait(Native Method) at java.lang.Thread.join(Thread.java:1252) at java.lang.Thread.join(Thread.java:1326) at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:313) at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:310) at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:491) 17) Thread[id=7344, name=qtp1661555970-7344, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=7336, name=qtp1329013312-7336, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=7340, name=qtp1329013312-7340, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=7357, name=qtp1329013312-7357, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=7358, name=qtp1661555970-7358, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 22) Thread[id=7349, name=qtp1661555970-7349, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=7345, name=qtp1661555970-7345, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=7346, name=qtp1661555970-7346, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 25) Thread[id=7352, name=Scheduler-2081218491, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=7338, name=qtp1329013312-7338, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=7341, name=Scheduler-959118455, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 28) Thread[id=7356, name=qtp1329013312-7356, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 28 threads leaked from SUITE scope at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest:
1) Thread[id=7335, name=qtp1329013312-7335-acceptor-0@ac94e40-ServerConnector@247acfac{HTTP/1.1,[http/1.1]}{127.0.0.1:58786}, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=7363, name=qtp1661555970-7363, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=7327, name=ProcessThread(sid:0 cport:41409):, state=WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:123)
4) Thread[id=7347, name=qtp1661555970-7347, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=7325, name=SessionTracker, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at java.lang.Object.wait(Native Method)
at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:147)
6) Thread[id=7355, name=qtp1329013312-7355, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=7324, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:196)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=7348, name=qtp1661555970-7348, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=7334, name=qtp1329013312-7334, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=7360, name=qtp1661555970-7360, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=7364, name=qtp1661555970-7364, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=7350, name=qtp1661555970-7350-acceptor-0@1bfd09c6-ServerConnector@3c4c0ab3{HTTP/1.1,[http/1.1]}{127.0.0.1:41409}, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=7361, name=qtp1661555970-7361, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=7333, name=qtp1329013312-7333, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$28/664856532.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
15) Thread[id=7362, name=Scheduler-1842259731, state=WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
16) Thread[id=7323, name=Thread-2090, state=WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at java.lang.Object.wait(Native Method)
at java.lang.Thread.join(Thread.java:1252)
at java.lang.Thread.join(Thread.java:1326)
at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:313)
at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:310)
at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:491)
17) Thread[id=7344, name=qtp1661555970-7344, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
18) Thread[id=7336, name=qtp1329013312-7336, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
19) Thread[id=7340, name=qtp1329013312-7340, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
20) Thread[id=7357, name=qtp1329013312-7357, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
21) Thread[id=7358, name=qtp1661555970-7358, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
22) Thread[id=7349, name=qtp1661555970-7349, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
23) Thread[id=7345, name=qtp1661555970-7345, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
24) Thread[id=7346, name=qtp1661555970-7346, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
25) Thread[id=7352, name=Scheduler-2081218491, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
26) Thread[id=7338, name=qtp1329013312-7338, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
27) Thread[id=7341, name=Scheduler-959118455, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
28) Thread[id=7356, name=qtp1329013312-7356, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([DF44788DBE20EF01]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=7333, name=qtp1329013312-7333, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.io.ManagedSelector$$Lambda$28/664856532.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=7362, name=Scheduler-1842259731, state=WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=7345, name=qtp1661555970-7345, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:109) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=7324, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method) at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300) at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207) at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213) at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:196) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=7352, name=Scheduler-2081218491, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=7348, name=qtp1661555970-7348, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=7341, name=Scheduler-959118455, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=7333, name=qtp1329013312-7333, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$28/664856532.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=7362, name=Scheduler-1842259731, state=WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=7345, name=qtp1661555970-7345, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.nio.ch.IOUtil.drain(Native Method)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:109)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=7324, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)
at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)
at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)
at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:196)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=7352, name=Scheduler-2081218491, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=7348, name=qtp1661555970-7348, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=7341, name=Scheduler-959118455, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([DF44788DBE20EF01]:0)
Build Log:
[...truncated 1837 lines...]
[junit4] JVM J1: stdout was not empty, see: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/lucene/build/core/test/temp/junit4-J1-20180315_231150_6945423026239327529097.sysout
[junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
[junit4] codec: CheapBastard, pf: LuceneVarGapFixedInterval, dvf: Memory
[junit4] <<< JVM J1: EOF ----
[...truncated 10810 lines...]
[junit4] Suite: org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest
[junit4] 2> 605087 INFO (SUITE-DeleteLastCustomShardedReplicaTest-seed#[DF44788DBE20EF01]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteLastCustomShardedReplicaTest_DF44788DBE20EF01-001/init-core-data-001
[junit4] 2> 605087 WARN (SUITE-DeleteLastCustomShardedReplicaTest-seed#[DF44788DBE20EF01]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
[junit4] 2> 605087 INFO (SUITE-DeleteLastCustomShardedReplicaTest-seed#[DF44788DBE20EF01]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 605089 INFO (SUITE-DeleteLastCustomShardedReplicaTest-seed#[DF44788DBE20EF01]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 605090 INFO (SUITE-DeleteLastCustomShardedReplicaTest-seed#[DF44788DBE20EF01]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteLastCustomShardedReplicaTest_DF44788DBE20EF01-001/tempDir-001
[junit4] 2> 605090 INFO (SUITE-DeleteLastCustomShardedReplicaTest-seed#[DF44788DBE20EF01]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 605090 INFO (Thread-2090) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 605090 INFO (Thread-2090) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 605091 ERROR (Thread-2090) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 605190 INFO (SUITE-DeleteLastCustomShardedReplicaTest-seed#[DF44788DBE20EF01]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:41409
[junit4] 2> 605193 INFO (zkConnectionManagerCallback-1666-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 605199 INFO (jetty-launcher-1663-thread-1) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T15:27:37-06:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 605200 INFO (jetty-launcher-1663-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 605200 INFO (jetty-launcher-1663-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 605200 INFO (jetty-launcher-1663-thread-1) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 605201 INFO (jetty-launcher-1663-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@18062087{/solr,null,AVAILABLE}
[junit4] 2> 605202 INFO (jetty-launcher-1663-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@247acfac{HTTP/1.1,[http/1.1]}{127.0.0.1:58786}
[junit4] 2> 605202 INFO (jetty-launcher-1663-thread-1) [ ] o.e.j.s.Server Started @607305ms
[junit4] 2> 605202 INFO (jetty-launcher-1663-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=58786}
[junit4] 2> 605203 ERROR (jetty-launcher-1663-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 605203 INFO (jetty-launcher-1663-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 605203 INFO (jetty-launcher-1663-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 605203 INFO (jetty-launcher-1663-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 605203 INFO (jetty-launcher-1663-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-15T23:37:05.398Z
[junit4] 2> 605205 INFO (jetty-launcher-1663-thread-2) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T15:27:37-06:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 605207 INFO (jetty-launcher-1663-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 605207 INFO (jetty-launcher-1663-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 605207 INFO (jetty-launcher-1663-thread-2) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 605207 INFO (jetty-launcher-1663-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@32173193{/solr,null,AVAILABLE}
[junit4] 2> 605208 INFO (jetty-launcher-1663-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3c4c0ab3{HTTP/1.1,[http/1.1]}{127.0.0.1:41409}
[junit4] 2> 605208 INFO (jetty-launcher-1663-thread-2) [ ] o.e.j.s.Server Started @607311ms
[junit4] 2> 605208 INFO (jetty-launcher-1663-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41409}
[junit4] 2> 605208 ERROR (jetty-launcher-1663-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 605208 INFO (jetty-launcher-1663-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 8.0.0
[junit4] 2> 605208 INFO (jetty-launcher-1663-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 605208 INFO (jetty-launcher-1663-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 605208 INFO (jetty-launcher-1663-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-03-15T23:37:05.403Z
[junit4] 2> 605214 WARN (qtp1661555970-7347) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
[junit4] 2> 605216 WARN (jetty-launcher-1663-thread-2-SendThread(127.0.0.1:41409)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> 605224 WARN (qtp1661555970-7345) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
[junit4] 2> 605225 WARN (jetty-launcher-1663-thread-1-SendThread(127.0.0.1:41409)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> 606349 WARN (qtp1661555970-7344) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
[junit4] 2> 606349 WARN (jetty-launcher-1663-thread-1-SendThread(127.0.0.1:41409)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> 607246 WARN (qtp1661555970-7345) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
[junit4] 2> 607247 WARN (jetty-launcher-1663-thread-2-SendThread(127.0.0.1:41409)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> 607903 WARN (qtp1661555970-7344) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
[junit4] 2> 607904 WARN (jetty-launcher-1663-thread-1-SendThread(127.0.0.1:41409)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> 609260 WARN (qtp1661555970-7345) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
[junit4] 2> 609260 WARN (jetty-launcher-1663-thread-2-SendThread(127.0.0.1:41409)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> 609773 WARN (qtp1661555970-7344) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
[junit4] 2> 609774 WARN (jetty-launcher-1663-thread-1-SendThread(127.0.0.1:41409)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> 610925 WARN (qtp1661555970-7345) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
[junit4] 2> 610926 WARN (jetty-launcher-1663-thread-2-SendThread(127.0.0.1:41409)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> 611678 WARN (qtp1661555970-7344) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
[junit4] 2> 611678 WARN (jetty-launcher-1663-thread-1-SendThread(127.0.0.1:41409)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> 612239 WARN (qtp1661555970-7345) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
[junit4] 2> 612239 WARN (jetty-launcher-1663-thread-2-SendThread(127.0.0.1:41409)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> 613081 WARN (qtp1661555970-7344) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
[junit4] 2> 613082 WARN (jetty-launcher-1663-thread-1-SendThread(127.0.0.1:41409)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> 614327 WARN (qtp1661555970-7345) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
[junit4] 2> 614327 WARN (jetty-launcher-1663-thread-1-SendThread(127.0.0.1:41409)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> 614335 WARN (qtp1661555970-7344) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
[junit4] 2> 614335 WARN (jetty-launcher-1663-thread-2-SendThread(127.0.0.1:41409)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> 615902 WARN (qtp1661555970-7345) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
[junit4] 2> 615902 WARN (jetty-launcher-1663-thread-1-SendThread(127.0.0.1:41409)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> 616396 WARN (qtp1661555970-7344) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
[junit4] 2> 616397 WARN (jetty-launcher-1663-thread-2-SendThread(127.0.0.1:41409)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> 617643 WARN (qtp1661555970-7345) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
[junit4] 2> 617643 WARN (jetty-launcher-1663-thread-2-SendThread(127.0.0.1:41409)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> 617893 WARN (qtp1661555970-7344) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00<<<\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}
[junit4] 2> 617894 WARN (jetty-launcher-1663-thread-1-SendThread(127.0.0.1:41409)) [ ] o.a.z.ClientCnxn Session 0x0 for server 127.0.0.1/127.0.0.1:41409, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.io.IOException: Packet len1213486160 is out of range!
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocket.readLength(ClientCnxnSocket.java:113)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:79)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> 619404 WARN (qtp1661555970-7345) [ ] o.e.j.h.HttpParser Illegal character 0x0 in state=START for buffer HeapByteBuffer@f923d2f[p=1,l=49,c=8192,r=48]={\x00
[...truncated too long message...]
[junit4] > at __randomizedtesting.SeedInfo.seed([DF44788DBE20EF01]:0)Throwable #3: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=7333, name=qtp1329013312-7333, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
[junit4] > at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
[junit4] > at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
[junit4] > at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$$Lambda$28/664856532.run(Unknown Source)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 2) Thread[id=7362, name=Scheduler-1842259731, state=WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 3) Thread[id=7345, name=qtp1661555970-7345, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
[junit4] > at sun.nio.ch.IOUtil.drain(Native Method)
[junit4] > at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:109)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
[junit4] > at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
[junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 4) Thread[id=7324, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-DeleteLastCustomShardedReplicaTest]
[junit4] > at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)
[junit4] > at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)
[junit4] > at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)
[junit4] > at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
[junit4] > at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
[junit4] > at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:196)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=7352, name=Scheduler-2081218491, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=7348, name=qtp1661555970-7348, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
[junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
[junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 7) Thread[id=7341, name=Scheduler-959118455, state=TIMED_WAITING, group=TGRP-DeleteLastCustomShardedReplicaTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([DF44788DBE20EF01]:0)
[junit4] Completed [149/785 (1!)] on J1 in 151.05s, 0 tests, 3 errors <<< FAILURES!
[...truncated 51531 lines...]
[repro] Jenkins log URL: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1735/consoleText
[repro] Revision: ac9559d70de5e2454e0515a08774890ad4012eef
[repro] Ant options: "-Dargs=-XX:-UseCompressedOops -XX:+UseParallelGC"
[repro] Repro line: ant test -Dtestcase=DeleteLastCustomShardedReplicaTest -Dtests.seed=DF44788DBE20EF01 -Dtests.slow=true -Dtests.locale=en-PH -Dtests.timezone=Europe/Malta -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[repro] ant clean
[...truncated 6 lines...]
[repro] Test suites by module:
[repro] solr/core
[repro] DeleteLastCustomShardedReplicaTest
[repro] ant compile-test
[...truncated 3313 lines...]
[repro] ant test-nocompile -Dtests.dups=5 -Dtests.maxfailures=5 -Dtests.class="*.DeleteLastCustomShardedReplicaTest" -Dtests.showOutput=onerror "-Dargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Dtests.seed=DF44788DBE20EF01 -Dtests.slow=true -Dtests.locale=en-PH -Dtests.timezone=Europe/Malta -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[...truncated 68 lines...]
[repro] Failures:
[repro] 0/5 failed: org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest
[repro] Exiting with code 0
[...truncated 70 lines...]