You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/11/05 04:36:15 UTC
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 282 -
Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/282/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
1 tests failed.
FAILED: org.apache.solr.core.TestDynamicLoading.testDynamicLoading
Error Message:
Stack Trace:
java.lang.AssertionError
at __randomizedtesting.SeedInfo.seed([3E658255FE87B737:E628AF02095A1297]:0)
at org.junit.Assert.fail(Assert.java:92)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNotNull(Assert.java:526)
at org.junit.Assert.assertNotNull(Assert.java:537)
at org.apache.solr.core.TestDynamicLoading.testDynamicLoading(TestDynamicLoading.java:116)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13430 lines...]
[junit4] Suite: org.apache.solr.core.TestDynamicLoading
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/init-core-data-001
[junit4] 2> 3268975 WARN (SUITE-TestDynamicLoading-seed#[3E658255FE87B737]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=3 numCloses=3
[junit4] 2> 3268976 INFO (SUITE-TestDynamicLoading-seed#[3E658255FE87B737]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 3268977 INFO (SUITE-TestDynamicLoading-seed#[3E658255FE87B737]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 3268977 INFO (SUITE-TestDynamicLoading-seed#[3E658255FE87B737]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 3268977 INFO (SUITE-TestDynamicLoading-seed#[3E658255FE87B737]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /d_gf/ay
[junit4] 2> 3268991 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3268991 INFO (Thread-7317) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3268991 INFO (Thread-7317) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3268994 ERROR (Thread-7317) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3269094 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.ZkTestServer start zk server on port:59570
[junit4] 2> 3269104 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f8a5f63a30000, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3269118 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 3269120 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 3269122 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 3269123 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 3269125 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 3269126 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 3269128 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 3269129 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 3269131 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 3269132 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 3269134 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 3269136 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 3269248 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3269249 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3a2ef02b{/d_gf/ay,null,AVAILABLE}
[junit4] 2> 3269250 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@f154e3c{HTTP/1.1,[http/1.1]}{127.0.0.1:60348}
[junit4] 2> 3269250 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.Server Started @3272164ms
[junit4] 2> 3269250 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/tempDir-001/control/data, hostContext=/d_gf/ay, hostPort=60348, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/control-001/cores}
[junit4] 2> 3269250 ERROR (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3269250 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 3269250 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3269250 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3269250 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-11-05T04:10:28.389Z
[junit4] 2> 3269254 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3269254 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/control-001/solr.xml
[junit4] 2> 3269257 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3269257 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3269257 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3269259 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59570/solr
[junit4] 2> 3269381 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3269382 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60348_d_gf%2Fay
[junit4] 2> 3269383 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.Overseer Overseer (id=98949859123593220-127.0.0.1:60348_d_gf%2Fay-n_0000000000) starting
[junit4] 2> 3269393 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60348_d_gf%2Fay
[junit4] 2> 3269398 INFO (OverseerStateUpdate-98949859123593220-127.0.0.1:60348_d_gf%2Fay-n_0000000000) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3269537 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d
[junit4] 2> 3269543 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d
[junit4] 2> 3269543 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d
[junit4] 2> 3269544 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/control-001/cores
[junit4] 2> 3269559 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3269560 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59570/solr ready
[junit4] 2> 3269566 INFO (qtp1092038236-29823) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:60348_d_gf%252Fay&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3269569 INFO (OverseerThreadFactory-12429-thread-1-processing-n:127.0.0.1:60348_d_gf%2Fay) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 3269677 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3269677 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3269783 INFO (zkCallback-4462-thread-1-processing-n:127.0.0.1:60348_d_gf%2Fay) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3270695 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 3270710 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 3270775 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3270793 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 3270793 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d
[junit4] 2> 3270793 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3270793 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 3270795 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 3270797 WARN (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3270834 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3270834 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3270835 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3270835 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3270836 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=73.4443359375, floorSegmentMB=0.9990234375, forceMergeDeletesPctAllowed=20.539385582008254, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 3270836 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@408867a7[control_collection_shard1_replica_n1] main]
[junit4] 2> 3270838 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3270838 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3270839 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3270839 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583197747914211328
[junit4] 2> 3270840 INFO (searcherExecutor-12432-thread-1-processing-n:127.0.0.1:60348_d_gf%2Fay x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@408867a7[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3270846 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3270846 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3270846 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60348/d_gf/ay/control_collection_shard1_replica_n1/
[junit4] 2> 3270846 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3270846 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:60348/d_gf/ay/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 3270846 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3270850 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60348/d_gf/ay/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 3270951 INFO (zkCallback-4462-thread-2-processing-n:127.0.0.1:60348_d_gf%2Fay) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3271001 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3271004 INFO (qtp1092038236-29824) [n:127.0.0.1:60348_d_gf%2Fay c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1327
[junit4] 2> 3271011 INFO (qtp1092038236-29823) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3271107 INFO (zkCallback-4462-thread-2-processing-n:127.0.0.1:60348_d_gf%2Fay) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3271573 INFO (OverseerCollectionConfigSetProcessor-98949859123593220-127.0.0.1:60348_d_gf%2Fay-n_0000000000) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3272012 INFO (qtp1092038236-29823) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:60348_d_gf%252Fay&wt=javabin&version=2} status=0 QTime=2446
[junit4] 2> 3272019 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3272020 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59570/solr ready
[junit4] 2> 3272021 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 3272021 INFO (qtp1092038236-29827) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3272024 INFO (OverseerThreadFactory-12429-thread-2-processing-n:127.0.0.1:60348_d_gf%2Fay) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 3272025 WARN (OverseerThreadFactory-12429-thread-2-processing-n:127.0.0.1:60348_d_gf%2Fay) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 3272232 INFO (qtp1092038236-29827) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3272232 INFO (qtp1092038236-29827) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=210
[junit4] 2> 3272349 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-1-001 of type NRT
[junit4] 2> 3272349 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3272350 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@43c943c0{/d_gf/ay,null,AVAILABLE}
[junit4] 2> 3272351 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@78f8b442{HTTP/1.1,[http/1.1]}{127.0.0.1:58222}
[junit4] 2> 3272351 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.Server Started @3275265ms
[junit4] 2> 3272351 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/d_gf/ay, hostPort=58222, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-1-001/cores}
[junit4] 2> 3272351 ERROR (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3272351 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 3272351 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3272351 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3272351 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-11-05T04:10:31.490Z
[junit4] 2> 3272355 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3272355 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-1-001/solr.xml
[junit4] 2> 3272357 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3272357 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3272358 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3272360 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59570/solr
[junit4] 2> 3272372 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3272376 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3272378 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58222_d_gf%2Fay
[junit4] 2> 3272381 INFO (zkCallback-4475-thread-1-processing-n:127.0.0.1:58222_d_gf%2Fay) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3272382 INFO (zkCallback-4462-thread-2-processing-n:127.0.0.1:60348_d_gf%2Fay) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3272381 INFO (zkCallback-4470-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3272432 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d
[junit4] 2> 3272440 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d
[junit4] 2> 3272440 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d
[junit4] 2> 3272441 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-1-001/cores
[junit4] 2> 3272501 INFO (qtp329181813-29869) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:58222_d_gf%252Fay&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3272504 INFO (OverseerCollectionConfigSetProcessor-98949859123593220-127.0.0.1:60348_d_gf%2Fay-n_0000000000) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3272505 INFO (OverseerThreadFactory-12429-thread-3-processing-n:127.0.0.1:60348_d_gf%2Fay) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:58222_d_gf%2Fay for creating new replica
[junit4] 2> 3272508 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3272509 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3272614 INFO (zkCallback-4475-thread-1-processing-n:127.0.0.1:58222_d_gf%2Fay) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3273523 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 3273531 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema [collection1_shard2_replica_n41] Schema name=test
[junit4] 2> 3273599 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3273616 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n41' using configuration from collection collection1, trusted=true
[junit4] 2> 3273617 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n41' (registry 'solr.core.collection1.shard2.replica_n41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d
[junit4] 2> 3273617 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3273617 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [[collection1_shard2_replica_n41] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-1-001/cores/collection1_shard2_replica_n41], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-1-001/cores/collection1_shard2_replica_n41/data/]
[junit4] 2> 3273618 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 3273620 WARN (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3273655 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3273655 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3273656 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3273656 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3273656 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=73.4443359375, floorSegmentMB=0.9990234375, forceMergeDeletesPctAllowed=20.539385582008254, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 3273657 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.SolrIndexSearcher Opening [Searcher@299898a2[collection1_shard2_replica_n41] main]
[junit4] 2> 3273658 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3273659 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3273659 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3273659 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583197750871195648
[junit4] 2> 3273660 INFO (searcherExecutor-12443-thread-1-processing-n:127.0.0.1:58222_d_gf%2Fay x:collection1_shard2_replica_n41 s:shard2 c:collection1) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [collection1_shard2_replica_n41] Registered new searcher Searcher@299898a2[collection1_shard2_replica_n41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3273665 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3273665 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3273665 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:58222/d_gf/ay/collection1_shard2_replica_n41/
[junit4] 2> 3273665 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3273665 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy http://127.0.0.1:58222/d_gf/ay/collection1_shard2_replica_n41/ has no replicas
[junit4] 2> 3273665 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3273669 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:58222/d_gf/ay/collection1_shard2_replica_n41/ shard2
[junit4] 2> 3273772 INFO (zkCallback-4475-thread-1-processing-n:127.0.0.1:58222_d_gf%2Fay) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3273821 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3273824 INFO (qtp329181813-29871) [n:127.0.0.1:58222_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1315
[junit4] 2> 3273826 INFO (qtp329181813-29869) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:58222_d_gf%252Fay&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1325
[junit4] 2> 3273925 INFO (zkCallback-4475-thread-1-processing-n:127.0.0.1:58222_d_gf%2Fay) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3273943 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-2-001 of type NRT
[junit4] 2> 3273944 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3273945 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4b51b95e{/d_gf/ay,null,AVAILABLE}
[junit4] 2> 3273945 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@277ecadc{HTTP/1.1,[http/1.1]}{127.0.0.1:33897}
[junit4] 2> 3273945 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.Server Started @3276859ms
[junit4] 2> 3273945 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/d_gf/ay, hostPort=33897, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-2-001/cores}
[junit4] 2> 3273946 ERROR (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3273946 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 3273946 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3273946 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3273946 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-11-05T04:10:33.085Z
[junit4] 2> 3273949 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3273949 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-2-001/solr.xml
[junit4] 2> 3273952 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3273952 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3273952 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3273954 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59570/solr
[junit4] 2> 3273969 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3273972 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3273974 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33897_d_gf%2Fay
[junit4] 2> 3273976 INFO (zkCallback-4475-thread-1-processing-n:127.0.0.1:58222_d_gf%2Fay) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3273976 INFO (zkCallback-4470-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3273976 INFO (zkCallback-4462-thread-2-processing-n:127.0.0.1:60348_d_gf%2Fay) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3273978 INFO (zkCallback-4481-thread-1-processing-n:127.0.0.1:33897_d_gf%2Fay) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3274234 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d
[junit4] 2> 3274241 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d
[junit4] 2> 3274241 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d
[junit4] 2> 3274242 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-2-001/cores
[junit4] 2> 3274270 INFO (qtp1555763870-29898) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:33897_d_gf%252Fay&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3274274 INFO (OverseerCollectionConfigSetProcessor-98949859123593220-127.0.0.1:60348_d_gf%2Fay-n_0000000000) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3274275 INFO (OverseerThreadFactory-12429-thread-4-processing-n:127.0.0.1:60348_d_gf%2Fay) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:33897_d_gf%2Fay for creating new replica
[junit4] 2> 3274278 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3274278 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3274384 INFO (zkCallback-4475-thread-1-processing-n:127.0.0.1:58222_d_gf%2Fay) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 3274384 INFO (zkCallback-4481-thread-1-processing-n:127.0.0.1:33897_d_gf%2Fay) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 3275293 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 3275302 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.IndexSchema [collection1_shard1_replica_n43] Schema name=test
[junit4] 2> 3275374 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3275390 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n43' using configuration from collection collection1, trusted=true
[junit4] 2> 3275391 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n43' (registry 'solr.core.collection1.shard1.replica_n43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d
[junit4] 2> 3275391 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3275391 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SolrCore [[collection1_shard1_replica_n43] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-2-001/cores/collection1_shard1_replica_n43], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-2-001/cores/collection1_shard1_replica_n43/data/]
[junit4] 2> 3275393 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 3275395 WARN (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3275429 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3275429 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3275430 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3275430 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3275431 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=73.4443359375, floorSegmentMB=0.9990234375, forceMergeDeletesPctAllowed=20.539385582008254, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 3275431 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.SolrIndexSearcher Opening [Searcher@26d609b[collection1_shard1_replica_n43] main]
[junit4] 2> 3275433 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3275433 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3275433 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3275434 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583197752732418048
[junit4] 2> 3275434 INFO (searcherExecutor-12454-thread-1-processing-n:127.0.0.1:33897_d_gf%2Fay x:collection1_shard1_replica_n43 s:shard1 c:collection1) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SolrCore [collection1_shard1_replica_n43] Registered new searcher Searcher@26d609b[collection1_shard1_replica_n43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3275439 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3275439 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3275439 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:33897/d_gf/ay/collection1_shard1_replica_n43/
[junit4] 2> 3275439 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3275439 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SyncStrategy http://127.0.0.1:33897/d_gf/ay/collection1_shard1_replica_n43/ has no replicas
[junit4] 2> 3275439 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3275443 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:33897/d_gf/ay/collection1_shard1_replica_n43/ shard1
[junit4] 2> 3275545 INFO (zkCallback-4475-thread-1-processing-n:127.0.0.1:58222_d_gf%2Fay) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 3275546 INFO (zkCallback-4481-thread-1-processing-n:127.0.0.1:33897_d_gf%2Fay) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 3275595 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3275597 INFO (qtp1555763870-29900) [n:127.0.0.1:33897_d_gf%2Fay c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1319
[junit4] 2> 3275601 INFO (qtp1555763870-29898) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:33897_d_gf%252Fay&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1331
[junit4] 2> 3275699 INFO (zkCallback-4475-thread-1-processing-n:127.0.0.1:58222_d_gf%2Fay) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 3275699 INFO (zkCallback-4481-thread-1-processing-n:127.0.0.1:33897_d_gf%2Fay) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 3275731 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-3-001 of type NRT
[junit4] 2> 3275732 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3275733 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3beab35e{/d_gf/ay,null,AVAILABLE}
[junit4] 2> 3275733 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3081ace6{HTTP/1.1,[http/1.1]}{127.0.0.1:55507}
[junit4] 2> 3275734 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.Server Started @3278648ms
[junit4] 2> 3275734 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/d_gf/ay, hostPort=55507, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-3-001/cores}
[junit4] 2> 3275734 ERROR (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3275734 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 3275734 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3275734 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3275734 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-11-05T04:10:34.873Z
[junit4] 2> 3275738 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3275738 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-3-001/solr.xml
[junit4] 2> 3275740 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3275740 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3275741 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3275743 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59570/solr
[junit4] 2> 3275757 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:55507_d_gf%2Fay ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 3275760 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:55507_d_gf%2Fay ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3275762 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:55507_d_gf%2Fay ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55507_d_gf%2Fay
[junit4] 2> 3275764 INFO (zkCallback-4470-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3275764 INFO (zkCallback-4481-thread-1-processing-n:127.0.0.1:33897_d_gf%2Fay) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3275765 INFO (zkCallback-4462-thread-2-processing-n:127.0.0.1:60348_d_gf%2Fay) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3275765 INFO (zkCallback-4475-thread-1-processing-n:127.0.0.1:58222_d_gf%2Fay) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3275767 INFO (zkCallback-4487-thread-1-processing-n:127.0.0.1:55507_d_gf%2Fay) [n:127.0.0.1:55507_d_gf%2Fay ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3275832 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:55507_d_gf%2Fay ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d
[junit4] 2> 3275840 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:55507_d_gf%2Fay ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d
[junit4] 2> 3275840 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:55507_d_gf%2Fay ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d
[junit4] 2> 3275841 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [n:127.0.0.1:55507_d_gf%2Fay ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-3-001/cores
[junit4] 2> 3275869 INFO (qtp1555763870-29901) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:55507_d_gf%252Fay&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3275873 INFO (OverseerCollectionConfigSetProcessor-98949859123593220-127.0.0.1:60348_d_gf%2Fay-n_0000000000) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3275874 INFO (OverseerThreadFactory-12429-thread-5-processing-n:127.0.0.1:60348_d_gf%2Fay) [n:127.0.0.1:60348_d_gf%2Fay ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:55507_d_gf%2Fay for creating new replica
[junit4] 2> 3275882 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3275882 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3275988 INFO (zkCallback-4481-thread-1-processing-n:127.0.0.1:33897_d_gf%2Fay) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 3275988 INFO (zkCallback-4475-thread-1-processing-n:127.0.0.1:58222_d_gf%2Fay) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 3275989 INFO (zkCallback-4487-thread-1-processing-n:127.0.0.1:55507_d_gf%2Fay) [n:127.0.0.1:55507_d_gf%2Fay ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 3276908 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 3276919 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.s.IndexSchema [collection1_shard2_replica_n45] Schema name=test
[junit4] 2> 3276991 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3277018 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n45' using configuration from collection collection1, trusted=true
[junit4] 2> 3277019 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n45' (registry 'solr.core.collection1.shard2.replica_n45') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7554e26d
[junit4] 2> 3277019 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3277019 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.SolrCore [[collection1_shard2_replica_n45] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-3-001/cores/collection1_shard2_replica_n45], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-3-001/cores/collection1_shard2_replica_n45/data/]
[junit4] 2> 3277020 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 3277023 WARN (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3277057 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3277057 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3277058 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3277058 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3277059 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=73.4443359375, floorSegmentMB=0.9990234375, forceMergeDeletesPctAllowed=20.539385582008254, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 3277059 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.s.SolrIndexSearcher Opening [Searcher@677a8e3f[collection1_shard2_replica_n45] main]
[junit4] 2> 3277061 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3277062 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3277062 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3277063 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1583197754440548352
[junit4] 2> 3277063 INFO (searcherExecutor-12465-thread-1-processing-n:127.0.0.1:55507_d_gf%2Fay x:collection1_shard2_replica_n45 s:shard2 c:collection1) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.SolrCore [collection1_shard2_replica_n45] Registered new searcher Searcher@677a8e3f[collection1_shard2_replica_n45] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3277067 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.ZkController Core needs to recover:collection1_shard2_replica_n45
[junit4] 2> 3277067 INFO (updateExecutor-4484-thread-1-processing-n:127.0.0.1:55507_d_gf%2Fay x:collection1_shard2_replica_n45 s:shard2 c:collection1) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 3277068 INFO (recoveryExecutor-4485-thread-1-processing-n:127.0.0.1:55507_d_gf%2Fay x:collection1_shard2_replica_n45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 3277068 INFO (recoveryExecutor-4485-thread-1-processing-n:127.0.0.1:55507_d_gf%2Fay x:collection1_shard2_replica_n45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 3277068 INFO (recoveryExecutor-4485-thread-1-processing-n:127.0.0.1:55507_d_gf%2Fay x:collection1_shard2_replica_n45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard2_replica_n45]
[junit4] 2> 3277068 INFO (recoveryExecutor-4485-thread-1-processing-n:127.0.0.1:55507_d_gf%2Fay x:collection1_shard2_replica_n45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 3277068 INFO (recoveryExecutor-4485-thread-1-processing-n:127.0.0.1:55507_d_gf%2Fay x:collection1_shard2_replica_n45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard2_replica_n45] as recovering, leader is [http://127.0.0.1:58222/d_gf/ay/collection1_shard2_replica_n41/] and I am [http://127.0.0.1:55507/d_gf/ay/collection1_shard2_replica_n45/]
[junit4] 2> 3277069 INFO (qtp96465274-29927) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1186
[junit4] 2> 3277072 INFO (qtp1555763870-29901) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:55507_d_gf%252Fay&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1202
[junit4] 2> 3277073 INFO (recoveryExecutor-4485-thread-1-processing-n:127.0.0.1:55507_d_gf%2Fay x:collection1_shard2_replica_n45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:55507_d_gf%2Fay c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:58222/d_gf/ay]; [WaitForState: action=PREPRECOVERY&core=collection1_shard2_replica_n41&nodeName=127.0.0.1:55507_d_gf%252Fay&coreNodeName=core_node46&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 3277074 INFO (qtp329181813-29872) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node46, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true, maxTime: 183 s
[junit4] 2> 3277074 INFO (qtp329181813-29872) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1_shard2_replica_n41, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:55507_d_gf%2Fay, coreNodeName=core_node46, onlyIfActiveCheckResult=false, nodeProps: core_node46:{"core":"collection1_shard2_replica_n45","base_url":"http://127.0.0.1:55507/d_gf/ay","node_name":"127.0.0.1:55507_d_gf%2Fay","state":"down","type":"NRT"}
[junit4] 2> 3277171 INFO (zkCallback-4481-thread-1-processing-n:127.0.0.1:33897_d_gf%2Fay) [n:127.0.0.1:33897_d_gf%2Fay ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 3277171 INFO (zkCallback-4487-thread-1-processing-n:127.0.0.1:55507_d_gf%2Fay) [n:127.0.0.1:55507_d_gf%2Fay ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 3277172 INFO (zkCallback-4475-thread-1-processing-n:127.0.0.1:58222_d_gf%2Fay) [n:127.0.0.1:58222_d_gf%2Fay ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 3277196 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-4-001 of type NRT
[junit4] 2> 3277196 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3277197 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1bd5389e{/d_gf/ay,null,AVAILABLE}
[junit4] 2> 3277198 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4186a15e{HTTP/1.1,[http/1.1]}{127.0.0.1:39172}
[junit4] 2> 3277198 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.e.j.s.Server Started @3280112ms
[junit4] 2> 3277198 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/tempDir-001/jetty4, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/d_gf/ay, hostPort=39172, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-4-001/cores}
[junit4] 2> 3277198 ERROR (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3277198 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.2.0
[junit4] 2> 3277198 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3277198 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3277198 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-11-05T04:10:36.337Z
[junit4] 2> 3277202 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3277202 INFO (TEST-TestDynamicLoading.testDynamicLoading-seed#[3E658255FE87B737]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001/shard-4-001/solr.xml
[junit4] 2> 3277204 INFO (TEST-
[...truncated too long message...]
9570 59570
[junit4] 2> 3288450 WARN (Thread-7317) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 15 /solr/configs/conf1
[junit4] 2> 7 /solr/aliases.json
[junit4] 2> 5 /solr/security.json
[junit4] 2> 2 /solr/configs/.system
[junit4] 2> 2 /solr/configs/.system/managed-schema
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/clusterprops.json
[junit4] 2> 7 /solr/clusterstate.json
[junit4] 2> 4 /solr/collections/collection1/state.json
[junit4] 2> 3 /solr/autoscaling.json
[junit4] 2> 2 /solr/collections/.system/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 7 /solr/live_nodes
[junit4] 2> 7 /solr/collections
[junit4] 2> 3 /solr/overseer/queue
[junit4] 2> 3 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 3 /solr/overseer/collection-queue-work
[junit4] 2> 3 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestDynamicLoading -Dtests.method=testDynamicLoading -Dtests.seed=3E658255FE87B737 -Dtests.slow=true -Dtests.locale=he-IL -Dtests.timezone=Asia/Vladivostok -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 19.5s J0 | TestDynamicLoading.testDynamicLoading <<<
[junit4] > Throwable #1: java.lang.AssertionError
[junit4] > at __randomizedtesting.SeedInfo.seed([3E658255FE87B737:E628AF02095A1297]:0)
[junit4] > at org.apache.solr.core.TestDynamicLoading.testDynamicLoading(TestDynamicLoading.java:116)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading_3E658255FE87B737-001
[junit4] 2> Nov 05, 2017 4:10:47 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1768, maxMBSortInHeap=5.5867698532906935, sim=RandomSimilarity(queryNorm=false): {}, locale=he-IL, timezone=Asia/Vladivostok
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=1,free=313290208,total=495124480
[junit4] 2> NOTE: All tests run in this JVM: [LeaderFailureAfterFreshStartTest, SolrCLIZkUtilsTest, SolrTestCaseJ4Test, ConfigSetsAPITest, TestGeoJSONResponseWriter, HdfsCollectionsAPIDistributedZkTest, TestReloadAndDeleteDocs, TestInitQParser, TestCloudPseudoReturnFields, TestTolerantUpdateProcessorRandomCloud, TestReRankQParserPlugin, TestHighFrequencyDictionaryFactory, AlternateDirectoryTest, SpatialRPTFieldTypeTest, DeleteShardTest, QueryParsingTest, CdcrReplicationHandlerTest, TestMultiValuedNumericRangeQuery, HdfsAutoAddReplicasIntegrationTest, TestCloudDeleteByQuery, ImplicitSnitchTest, TestDistributedStatsComponentCardinality, TestPrepRecovery, CleanupOldIndexTest, TestAuthorizationFramework, NoCacheHeaderTest, TestRequestStatusCollectionAPI, HdfsBasicDistributedZk2Test, TestUninvertingReader, ReplicationFactorTest, TestReloadDeadlock, URLClassifyProcessorTest, SignatureUpdateProcessorFactoryTest, TestLFUCache, TestTrackingShardHandlerFactory, SparseHLLTest, TestSurroundQueryParser, TestReplicationHandler, BasicDistributedZkTest, TestCloudJSONFacetJoinDomain, TestChildDocTransformer, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestStreamBody, TestConfigSetImmutable, TestHttpShardHandlerFactory, TestCustomDocTransformer, DistributedTermsComponentTest, TestReplicaProperties, DirectoryFactoryTest, DistributedDebugComponentTest, TestStressUserVersions, TestQuerySenderNoQuery, SolrMetricsIntegrationTest, XsltUpdateRequestHandlerTest, TestLegacyNumericUtils, TestStressInPlaceUpdates, TestJoin, PropertiesRequestHandlerTest, TestCloudNestedDocsSort, HLLSerializationTest, SolrMetricManagerTest, TestOmitPositions, SyncSliceTest, ZkCLITest, CloneFieldUpdateProcessorFactoryTest, TestCSVResponseWriter, PathHierarchyTokenizerFactoryTest, TestMultiWordSynonyms, TestCustomStream, TestMiniSolrCloudClusterSSL, DisMaxRequestHandlerTest, V2ApiIntegrationTest, TestConfigsApi, TestNumericTerms32, OverriddenZkACLAndCredentialsProvidersTest, TestCopyFieldCollectionResource, AutoAddReplicasPlanActionTest, TestStressRecovery, TestRecovery, SolrJmxReporterTest, CollectionsAPISolrJTest, TestManagedSchema, CreateCollectionCleanupTest, TestIndexingPerformance, TestJsonRequest, OutputWriterTest, TestDefaultStatsCache, CircularListTest, LeaderElectionTest, TestSolrCoreProperties, TestFieldCacheReopen, TestSolrXml, BadIndexSchemaTest, TestUpdate, BlockJoinFacetDistribTest, TestTolerantSearch, AtomicUpdatesTest, CursorMarkTest, AddReplicaTest, SortByFunctionTest, TestLazyCores, SegmentsInfoRequestHandlerTest, TestTlogReplica, DateRangeFieldTest, InfoHandlerTest, TestSolr4Spatial2, TestPhraseSuggestions, CollectionReloadTest, TestUseDocValuesAsStored, TestCodecSupport, ClusterStateUpdateTest, TestCloudSearcherWarming, HdfsRestartWhileUpdatingTest, SpellCheckCollatorTest, SortSpecParsingTest, BlockJoinFacetSimpleTest, DeleteInactiveReplicaTest, IndexSchemaTest, FieldAnalysisRequestHandlerTest, TestGraphTermsQParserPlugin, TestRandomDVFaceting, TestJavabinTupleStreamParser, HdfsDirectoryFactoryTest, TestPseudoReturnFields, PrimitiveFieldTypeTest, TestDelegationWithHadoopAuth, HdfsChaosMonkeyNothingIsSafeTest, TestRealTimeGet, TestXmlQParser, BigEndianAscendingWordSerializerTest, MultiTermTest, OverseerTest, TestIBSimilarityFactory, TestFuzzyAnalyzedSuggestions, ClassificationUpdateProcessorIntegrationTest, TestImplicitCoreProperties, TriLevelCompositeIdRoutingTest, TestInitParams, CursorPagingTest, DateFieldTest, TestCursorMarkWithoutUniqueKey, MoveReplicaTest, ReturnFieldsTest, TestExactSharedStatsCache, ZkControllerTest, TestConfigSets, TestMacros, MoveReplicaHDFSFailoverTest, RulesTest, HdfsUnloadDistributedZkTest, TestFieldCacheSortRandom, UpdateLogTest, SpellCheckComponentTest, SuggesterFSTTest, TestJsonFacetRefinement, TestQuerySenderListener, RAMDirectoryFactoryTest, DistributedQueryElevationComponentTest, DistributedQueryComponentCustomSortTest, WrapperMergePolicyFactoryTest, DistributedFacetPivotSmallTest, TestRTimerTree, TestTestInjection, TestSimpleTextCodec, ChaosMonkeySafeLeaderWithPullReplicasTest, ExplicitHLLTest, ConnectionReuseTest, AnalyticsMergeStrategyTest, SolrCmdDistributorTest, TestConfigSetsAPI, TestCollectionAPI, TestFileDictionaryLookup, TestPayloadScoreQParserPlugin, DistribCursorPagingTest, TestReversedWildcardFilterFactory, TestReplicationHandlerBackup, NodeMutatorTest, TestPHPSerializedResponseWriter, TestConfigSetsAPIZkFailure, XmlUpdateRequestHandlerTest, SolrCloudReportersTest, SaslZkACLProviderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, OpenCloseCoreStressTest, LeaderElectionIntegrationTest, ShardRoutingTest, BasicZkTest, RecoveryZkTest, ShardRoutingCustomTest, TestDistributedSearch, TestFaceting, TestHashPartitioner, DistributedSpellCheckComponentTest, TestReload, TestCoreContainer, SimpleFacetsTest, TestSolr4Spatial, SolrCoreTest, ConvertedLegacyTest, TestSort, TestFiltering, TestFunctionQuery, DirectUpdateHandlerTest, HighlighterTest, ShowFileRequestHandlerTest, SolrIndexSplitterTest, SimplePostToolTest, TestCoreDiscovery, AnalysisAfterCoreReloadTest, TestExtendedDismaxParser, SolrRequestParserTest, SuggesterTSTTest, TestTrie, SuggesterWFSTTest, WordBreakSolrSpellCheckerTest, SchemaVersionSpecificBehaviorTest, DirectUpdateHandlerOptimizeTest, StatelessScriptUpdateProcessorFactoryTest, SolrInfoBeanTest, TestWriterPerf, RequestHandlersTest, TermVectorComponentTest, MoreLikeThisHandlerTest, FastVectorHighlighterTest, RegexBoostProcessorTest, TestCollationField, TestAnalyzedSuggestions, BinaryUpdateRequestHandlerTest, PingRequestHandlerTest, HighlighterConfigTest, SOLR749Test, ResponseLogComponentTest, TestSolrDeletionPolicy2, TestSearchPerf, TestElisionMultitermQuery, TestPostingsSolrHighlighter, DirectSolrConnectionTest, TestCharFilters, TestXIncludeConfig, TestDFRSimilarityFactory, TestLMDirichletSimilarityFactory, TestFastWriter, ResourceLoaderTest, OpenExchangeRatesOrgProviderTest, PluginInfoTest, ChaosMonkeyNothingIsSafeTest, AssignTest, CdcrRequestHandlerTest, CdcrVersionReplicationTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, DistributedQueueTest, DocValuesNotIndexedTest, HttpPartitionTest, LeaderElectionContextKeyTest, LeaderInitiatedRecoveryOnCommitTest, MissingSegmentRecoveryTest, MoveReplicaHDFSTest, OverseerTaskQueueTest, RecoveryAfterSoftCommitTest, SSLMigrationTest, SharedFSAutoReplicaFailoverTest, SimpleCollectionCreateDeleteTest, SolrXmlInZkTest, TestCollectionsAPIViaSolrCloudCluster, TestDeleteCollectionOnDownNodes, TestHdfsCloudBackupRestore, TestLocalFSCloudBackupRestore, TestRandomRequestDistribution, TestRebalanceLeaders, TestSegmentSorting, TestSizeLimitedDistributedMap, ZkFailoverTest, HdfsChaosMonkeySafeLeaderTest, HdfsRecoverLeaseTest, HdfsSyncSliceTest, HdfsThreadLeakTest, HdfsTlogReplayBufferedWhileIndexingTest, StressHdfsTest, RuleEngineTest, BlobRepositoryCloudTest, ConfigureRecoveryStrategyTest, TestBackupRepositoryFactory, TestConfigOverlay, TestConfigSetProperties, TestCorePropertiesReload, TestDynamicLoading]
[junit4] Completed [576/747 (1!)] on J0 in 20.05s, 1 test, 1 failure <<< FAILURES!
[...truncated 47829 lines...]
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 284 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/284/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
1 tests failed.
FAILED: org.apache.solr.util.UtilsToolTest.testRelativePath
Error Message:
expected:<14> but was:<15>
Stack Trace:
java.lang.AssertionError: expected:<14> but was:<15>
at __randomizedtesting.SeedInfo.seed([51F357FF9A4D8FB:F1835240E65829CA]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.util.UtilsToolTest.testRelativePath(UtilsToolTest.java:124)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12310 lines...]
[junit4] Suite: org.apache.solr.util.UtilsToolTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.util.UtilsToolTest_51F357FF9A4D8FB-001/init-core-data-001
[junit4] 2> 2409891 WARN (SUITE-UtilsToolTest-seed#[51F357FF9A4D8FB]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=46 numCloses=46
[junit4] 2> 2409891 INFO (SUITE-UtilsToolTest-seed#[51F357FF9A4D8FB]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 2409892 INFO (SUITE-UtilsToolTest-seed#[51F357FF9A4D8FB]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 2409892 INFO (SUITE-UtilsToolTest-seed#[51F357FF9A4D8FB]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 2409898 INFO (TEST-UtilsToolTest.testRemoveOldSolrLogs-seed#[51F357FF9A4D8FB]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testRemoveOldSolrLogs
[junit4] 1> Deleting 1 solr_log_* files older than 1 days.
[junit4] 1> Deleting 1 solr_log_* files older than 1 days.
[junit4] 2> 2409912 INFO (TEST-UtilsToolTest.testRemoveOldSolrLogs-seed#[51F357FF9A4D8FB]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testRemoveOldSolrLogs
[junit4] 2> 2409919 INFO (TEST-UtilsToolTest.testRemoveOldGcLogs-seed#[51F357FF9A4D8FB]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testRemoveOldGcLogs
[junit4] 1> Archiving 5 old GC log files to /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.util.UtilsToolTest_51F357FF9A4D8FB-001/Utils Tool Test-001/archived
[junit4] 2> 2409925 INFO (TEST-UtilsToolTest.testRemoveOldGcLogs-seed#[51F357FF9A4D8FB]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testRemoveOldGcLogs
[junit4] 2> 2409931 INFO (TEST-UtilsToolTest.testRotateSolrLogs-seed#[51F357FF9A4D8FB]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testRotateSolrLogs
[junit4] 1> Rotating solr logs, keeping a max of 9 generations
[junit4] 2> 2409939 INFO (TEST-UtilsToolTest.testRotateSolrLogs-seed#[51F357FF9A4D8FB]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testRotateSolrLogs
[junit4] 2> 2409947 INFO (TEST-UtilsToolTest.testEmptyAndQuiet-seed#[51F357FF9A4D8FB]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testEmptyAndQuiet
[junit4] 2> 2409952 INFO (TEST-UtilsToolTest.testEmptyAndQuiet-seed#[51F357FF9A4D8FB]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testEmptyAndQuiet
[junit4] 2> 2409961 INFO (TEST-UtilsToolTest.testRelativePath-seed#[51F357FF9A4D8FB]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testRelativePath
[junit4] 1> Deleting 1 solr_log_* files older than 0 days.
[junit4] 2> 2409965 INFO (TEST-UtilsToolTest.testRelativePath-seed#[51F357FF9A4D8FB]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testRelativePath
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=UtilsToolTest -Dtests.method=testRelativePath -Dtests.seed=51F357FF9A4D8FB -Dtests.slow=true -Dtests.locale=zh-HK -Dtests.timezone=Pacific/Wallis -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 0.01s J1 | UtilsToolTest.testRelativePath <<<
[junit4] > Throwable #1: java.lang.AssertionError: expected:<14> but was:<15>
[junit4] > at __randomizedtesting.SeedInfo.seed([51F357FF9A4D8FB:F1835240E65829CA]:0)
[junit4] > at org.apache.solr.util.UtilsToolTest.testRelativePath(UtilsToolTest.java:124)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2409974 INFO (TEST-UtilsToolTest.testArchiveConsoleLogs-seed#[51F357FF9A4D8FB]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testArchiveConsoleLogs
[junit4] 1> Archiving 1 console log files to /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.util.UtilsToolTest_51F357FF9A4D8FB-001/Utils Tool Test-001/archived
[junit4] 2> 2409978 INFO (TEST-UtilsToolTest.testArchiveConsoleLogs-seed#[51F357FF9A4D8FB]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testArchiveConsoleLogs
[junit4] 2> 2409985 INFO (TEST-UtilsToolTest.testNonexisting-seed#[51F357FF9A4D8FB]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNonexisting
[junit4] 2> 2409987 INFO (TEST-UtilsToolTest.testNonexisting-seed#[51F357FF9A4D8FB]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNonexisting
[junit4] 2> 2409995 INFO (TEST-UtilsToolTest.testRelativePathError-seed#[51F357FF9A4D8FB]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testRelativePathError
[junit4] 2> 2409997 INFO (TEST-UtilsToolTest.testRelativePathError-seed#[51F357FF9A4D8FB]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testRelativePathError
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.util.UtilsToolTest_51F357FF9A4D8FB-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1616, maxMBSortInHeap=7.1453749171227035, sim=RandomSimilarity(queryNorm=false): {}, locale=zh-HK, timezone=Pacific/Wallis
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=1,free=88302928,total=518979584
[junit4] 2> NOTE: All tests run in this JVM: [DistributedExpandComponentTest, TestManagedSynonymGraphFilterFactory, TestFieldCollectionResource, TestLeaderInitiatedRecoveryThread, BasicFunctionalityTest, SoftAutoCommitTest, TestFieldCacheVsDocValues, TestCloudInspectUtil, TestNamedUpdateProcessors, ReplicaListTransformerTest, HdfsChaosMonkeySafeLeaderTest, TestCollationField, SolrMetricReporterTest, TestTolerantUpdateProcessorCloud, TestRandomFaceting, BitVectorTest, TestSchemaSimilarityResource, TestFilteredDocIdSet, TestExtendedDismaxParser, TestNoOpRegenerator, TestObjectReleaseTracker, TestPayloadCheckQParserPlugin, TestExactStatsCache, CustomCollectionTest, RecoveryAfterSoftCommitTest, DirectSolrSpellCheckerTest, MultiThreadedOCPTest, ParsingFieldUpdateProcessorsTest, BlockJoinFacetRandomTest, TestJettySolrRunner, DocValuesNotIndexedTest, LoggingHandlerTest, TestRawTransformer, TestXmlQParserPlugin, TestNumericTokenStream, BufferStoreTest, TestGraphMLResponseWriter, ChaosMonkeyNothingIsSafeWithPullReplicasTest, TestDynamicLoading, TestBlendedInfixSuggestions, TestComplexPhraseLeadingWildcard, RequestLoggingTest, MoreLikeThisHandlerTest, BlobRepositoryCloudTest, ClassificationUpdateProcessorFactoryTest, IndexSchemaRuntimeFieldTest, SimpleCollectionCreateDeleteTest, SolrCoreTest, TestPostingsSolrHighlighter, TestStressReorder, NodeLostTriggerTest, TestRTGBase, TestQueryUtils, TestSolrConfigHandler, CopyFieldTest, SolrSlf4jReporterTest, PeerSyncTest, TestSolrQueryParser, TestNonDefinedSimilarityFactory, ActionThrottleTest, TestSuggestSpellingConverter, SynonymTokenizerTest, TestExclusionRuleCollectionAccess, HdfsTlogReplayBufferedWhileIndexingTest, DistributedFacetPivotWhiteBoxTest, DocumentBuilderTest, TestSolrQueryResponse, TestLuceneIndexBackCompat, TestSha256AuthenticationProvider, TestSolrCloudSnapshots, TestTrie, TestAuthenticationFramework, TestFieldTypeCollectionResource, ShardRoutingCustomTest, TestConfigOverlay, TestNestedDocsSort, TestHashPartitioner, PreAnalyzedFieldTest, TestStressLucene, TestShortCircuitedRequests, TestLegacyFieldReuse, AssignTest, TestInfoStreamLogging, TestSSLRandomization, JavabinLoaderTest, TestLegacyNumericRangeQueryBuilder, QueryElevationComponentTest, TermVectorComponentTest, TestLockTree, TestUnifiedSolrHighlighter, BasicZkTest, ZkNodePropsTest, ZkSolrClientTest, TestNumericRangeQuery32, PeerSyncWithIndexFingerprintCachingTest, TestFastOutputStream, SOLR749Test, TestDynamicFieldCollectionResource, TestCloudSchemaless, TlogReplayBufferedWhileIndexingTest, JsonLoaderTest, FileUtilsTest, TestSystemCollAutoCreate, TestSolrDeletionPolicy1, DeleteLastCustomShardedReplicaTest, TestCoreAdminApis, TestIntervalFaceting, CollectionStateFormat2Test, CachingDirectoryFactoryTest, TestPushWriter, JvmMetricsTest, ExitableDirectoryReaderTest, CdcrUpdateLogTest, UpdateParamsTest, TestQueryTypes, HdfsNNFailoverTest, TestStressCloudBlindAtomicUpdates, TestCloudRecovery, UtilsToolTest]
[junit4] Completed [264/747 (1!)] on J1 in 0.13s, 8 tests, 1 failure <<< FAILURES!
[...truncated 48963 lines...]
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 283 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/283/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
3 tests failed.
FAILED: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test
Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:33362/l
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:33362/l
at __randomizedtesting.SeedInfo.seed([4B8FFC78BF16BDBE:C3DBC3A211EAD046]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1096)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:875)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:808)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:183)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:200)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:418)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
... 44 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
Error Message:
96 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest: 1) Thread[id=21854, name=qtp621640665-21854, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=21957, name=zkCallback-2899-thread-11, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=21874, name=Scheduler-1348615465, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=21870, name=Thread-4923, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1717) 5) Thread[id=21903, name=Connection evictor, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=21813, name=qtp1242382532-21813-acceptor-0@63bb23b1-ServerConnector@4ba1af56{HTTP/1.1,[http/1.1]}{127.0.0.1:51284}, state=RUNNABLE, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=21788, name=org.eclipse.jetty.server.session.HashSessionManager@1b49817Timer, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=21838, name=Scheduler-1424828278, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=21962, name=zkCallback-2906-thread-6, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=21817, name=qtp1242382532-21817, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=21850, name=qtp621640665-21850, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=21960, name=zkCallback-2906-thread-4, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=21832, name=Thread-4914, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1717) 14) Thread[id=21882, name=searcherExecutor-8045-thread-1, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=21814, name=qtp1242382532-21814, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=21955, name=zkCallback-2899-thread-9, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=21786, name=qtp1314228837-21786, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=21825, name=Connection evictor, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=21847, name=Connection evictor, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=21798, name=TEST-LeaderInitiatedRecoveryOnCommitTest.test-seed#[4B8FFC78BF16BDBE]-EventThread, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 21) Thread[id=21952, name=zkCallback-2899-thread-6, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 22) Thread[id=21781, name=qtp1314228837-21781-acceptor-0@2f7fb807-ServerConnector@2d9a36b4{HTTP/1.1,[http/1.1]}{127.0.0.1:47002}, state=RUNNABLE, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=21782, name=qtp1314228837-21782, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=21893, name=qtp647167649-21893, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 25) Thread[id=21784, name=qtp1314228837-21784, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=21970, name=zkCallback-2913-thread-6, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=21785, name=qtp1314228837-21785, state=RUNNABLE, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 28) Thread[id=21862, name=Connection evictor, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 29) Thread[id=21894, name=qtp647167649-21894, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 30) Thread[id=21971, name=zkCallback-2913-thread-7, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 31) Thread[id=21895, name=qtp647167649-21895, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 32) Thread[id=21853, name=qtp621640665-21853, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 33) Thread[id=21947, name=zkCallback-2893-thread-7, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 34) Thread[id=21816, name=qtp1242382532-21816, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=21959, name=zkCallback-2906-thread-3, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=21868, name=TEST-LeaderInitiatedRecoveryOnCommitTest.test-seed#[4B8FFC78BF16BDBE]-EventThread, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 37) Thread[id=21888, name=Connection evictor, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 38) Thread[id=21904, name=Connection evictor, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 39) Thread[id=21793, name=Connection evictor, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 40) Thread[id=21867, name=TEST-LeaderInitiatedRecoveryOnCommitTest.test-seed#[4B8FFC78BF16BDBE]-SendThread(127.0.0.1:44828), state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230) at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1246) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1170) 41) Thread[id=21961, name=zkCallback-2906-thread-5, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 42) Thread[id=21968, name=zkCallback-2913-thread-4, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 43) Thread[id=21800, name=Thread-4905, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1717) 44) Thread[id=21819, name=qtp1242382532-21819, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 45) Thread[id=21852, name=qtp621640665-21852, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 46) Thread[id=21948, name=zkCallback-2899-thread-2, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 47) Thread[id=21855, name=qtp621640665-21855, state=RUNNABLE, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 48) Thread[id=21891, name=qtp647167649-21891, state=RUNNABLE, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 49) Thread[id=21811, name=Connection evictor, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 50) Thread[id=21783, name=qtp1314228837-21783, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 51) Thread[id=21851, name=qtp621640665-21851-acceptor-0@fe33b17-ServerConnector@7af131c7{HTTP/1.1,[http/1.1]}{127.0.0.1:53314}, state=RUNNABLE, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 52) Thread[id=21858, name=org.eclipse.jetty.server.session.HashSessionManager@2cb9e4dfTimer, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 53) Thread[id=21812, name=qtp1242382532-21812, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 54) Thread[id=21829, name=TEST-LeaderInitiatedRecoveryOnCommitTest.test-seed#[4B8FFC78BF16BDBE]-SendThread(127.0.0.1:44828), state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 55) Thread[id=21787, name=qtp1314228837-21787, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 56) Thread[id=21964, name=zkCallback-2906-thread-8, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 57) Thread[id=21896, name=qtp647167649-21896, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 58) Thread[id=21958, name=zkCallback-2906-thread-2, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 59) Thread[id=21897, name=qtp647167649-21897, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 60) Thread[id=21841, name=searcherExecutor-8034-thread-1, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 61) Thread[id=21944, name=zkCallback-2893-thread-4, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 62) Thread[id=21963, name=zkCallback-2906-thread-7, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 63) Thread[id=21898, name=qtp647167649-21898, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 64) Thread[id=21857, name=qtp621640665-21857, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 65) Thread[id=21856, name=qtp621640665-21856, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 66) Thread[id=21908, name=TEST-LeaderInitiatedRecoveryOnCommitTest.test-seed#[4B8FFC78BF16BDBE]-SendThread(127.0.0.1:44828), state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 67) Thread[id=21892, name=qtp647167649-21892-acceptor-0@5e8e0b67-ServerConnector@407b637d{HTTP/1.1,[http/1.1]}{127.0.0.1:37808}, state=RUNNABLE, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 68) Thread[id=21966, name=zkCallback-2906-thread-10, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 69) Thread[id=21797, name=TEST-LeaderInitiatedRecoveryOnCommitTest.test-seed#[4B8FFC78BF16BDBE]-SendThread(127.0.0.1:44828), state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 70) Thread[id=21909, name=TEST-LeaderInitiatedRecoveryOnCommitTest.test-seed#[4B8FFC78BF16BDBE]-EventThread, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 71) Thread[id=21943, name=zkCallback-2893-thread-3, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 72) Thread[id=21967, name=zkCallback-2913-thread-3, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 73) Thread[id=21824, name=Connection evictor, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 74) Thread[id=21815, name=qtp1242382532-21815, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 75) Thread[id=21951, name=zkCallback-2899-thread-5, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 76) Thread[id=21911, name=Thread-4932, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1717) 77) Thread[id=21809, name=searcherExecutor-8023-thread-1, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 78) Thread[id=21945, name=zkCallback-2893-thread-5, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 79) Thread[id=21950, name=zkCallback-2899-thread-4, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 80) Thread[id=21956, name=zkCallback-2899-thread-10, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 81) Thread[id=21899, name=org.eclipse.jetty.server.session.HashSessionManager@21e01b66Timer, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 82) Thread[id=21830, name=TEST-LeaderInitiatedRecoveryOnCommitTest.test-seed#[4B8FFC78BF16BDBE]-EventThread, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 83) Thread[id=21941, name=zkCallback-2893-thread-2-processing-n:127.0.0.1:58295_l, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:296) at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:475) at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:430) at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:417) at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:788) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56) at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348) at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$0(SolrZkClient.java:262) at org.apache.solr.common.cloud.SolrZkClient$1$$Lambda$211/2043215008.run(Unknown Source) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$117/546266590.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 84) Thread[id=21780, name=qtp1314228837-21780, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 85) Thread[id=21949, name=zkCallback-2899-thread-3, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 86) Thread[id=21915, name=searcherExecutor-8056-thread-1, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 87) Thread[id=21946, name=zkCallback-2893-thread-6, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 88) Thread[id=21969, name=zkCallback-2913-thread-5, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 89) Thread[id=21863, name=Connection evictor, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 90) Thread[id=21954, name=zkCallback-2899-thread-8, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 91) Thread[id=21820, name=org.eclipse.jetty.server.session.HashSessionManager@5e73599cTimer, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 92) Thread[id=21965, name=zkCallback-2906-thread-9, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 93) Thread[id=21953, name=zkCallback-2899-thread-7, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 94) Thread[id=21818, name=qtp1242382532-21818, state=RUNNABLE, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 95) Thread[id=21792, name=Connection evictor, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 96) Thread[id=21806, name=Scheduler-1116700810, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 96 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest:
1) Thread[id=21854, name=qtp621640665-21854, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=21957, name=zkCallback-2899-thread-11, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=21874, name=Scheduler-1348615465, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=21870, name=Thread-4923, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1717)
5) Thread[id=21903, name=Connection evictor, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=21813, name=qtp1242382532-21813-acceptor-0@63bb23b1-ServerConnector@4ba1af56{HTTP/1.1,[http/1.1]}{127.0.0.1:51284}, state=RUNNABLE, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=21788, name=org.eclipse.jetty.server.session.HashSessionManager@1b49817Timer, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=21838, name=Scheduler-1424828278, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=21962, name=zkCallback-2906-thread-6, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=21817, name=qtp1242382532-21817, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=21850, name=qtp621640665-21850, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=21960, name=zkCallback-2906-thread-4, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=21832, name=Thread-4914, state=WA
[...truncated too long message...]
ker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 45) Thread[id=21968, name=zkCallback-2913-thread-4, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 46) Thread[id=21954, name=zkCallback-2899-thread-8, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 47) Thread[id=21800, name=Thread-4905, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1717)
[junit4] > 48) Thread[id=21820, name=org.eclipse.jetty.server.session.HashSessionManager@5e73599cTimer, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 49) Thread[id=21965, name=zkCallback-2906-thread-9, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 50) Thread[id=21953, name=zkCallback-2899-thread-7, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 51) Thread[id=21948, name=zkCallback-2899-thread-2, state=TIMED_WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 52) Thread[id=21818, name=qtp1242382532-21818, state=RUNNABLE, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
[junit4] > at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)
[junit4] > at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)
[junit4] > at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)
[junit4] > at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
[junit4] > at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
[junit4] > at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 53) Thread[id=21806, name=Scheduler-1116700810, state=WAITING, group=TGRP-LeaderInitiatedRecoveryOnCommitTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([4B8FFC78BF16BDBE]:0)
[junit4] Completed [544/747 (1!)] on J0 in 358.18s, 1 test, 3 errors <<< FAILURES!
[...truncated 47962 lines...]