You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2016/06/04 02:37:21 UTC

[JENKINS] Lucene-Solr-6.x-Linux (64bit/jdk1.8.0_92) - Build # 825 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/825/
Java: 64bit/jdk1.8.0_92 -XX:+UseCompressedOops -XX:+UseSerialGC

2 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
There are still nodes recoverying - waited for 330 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 330 seconds
	at __randomizedtesting.SeedInfo.seed([CBA5C903E2D068E7:43F1F6D94C2C051F]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:182)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:134)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:850)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:124)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.DeleteShardTest.test

Error Message:
No registered leader was found after waiting for 10000ms , collection: collection1 slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 10000ms , collection: collection1 slice: shard1
	at __randomizedtesting.SeedInfo.seed([CBA5C903E2D068E7:43F1F6D94C2C051F]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:716)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:427)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:332)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:990)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11290 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/init-core-data-001
   [junit4]   2> 201500 INFO  (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[CBA5C903E2D068E7]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 201500 INFO  (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[CBA5C903E2D068E7]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 201502 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 201502 INFO  (Thread-689) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 201502 INFO  (Thread-689) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 201602 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.ZkTestServer start zk server on port:45093
   [junit4]   2> 201602 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 201602 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 201604 INFO  (zkCallback-407-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@c35e946 name:ZooKeeperConnection Watcher:127.0.0.1:45093 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 201604 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 201604 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 201604 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 201606 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 201607 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 201607 INFO  (zkCallback-408-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@756f00ce name:ZooKeeperConnection Watcher:127.0.0.1:45093/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 201607 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 201608 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 201608 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 201608 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards
   [junit4]   2> 201609 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection
   [junit4]   2> 201609 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards
   [junit4]   2> 201610 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 201610 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 201614 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 201614 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
   [junit4]   2> 201616 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 201616 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 201617 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 201617 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 201618 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 201618 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt
   [junit4]   2> 201619 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 201619 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml
   [junit4]   2> 201619 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 201619 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 201620 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 201620 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 201620 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 201621 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 201621 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 201621 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 201622 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 201622 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 201777 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001/cores/collection1
   [junit4]   2> 201778 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 201779 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@12d9de95{/,null,AVAILABLE}
   [junit4]   2> 201781 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.e.j.s.ServerConnector Started ServerConnector@7cd24528{HTTP/1.1,[http/1.1]}{127.0.0.1:36798}
   [junit4]   2> 201781 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.e.j.s.Server Started @203561ms
   [junit4]   2> 201781 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/tempDir-001/control/data, hostContext=/, hostPort=36798, coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001/cores}
   [junit4]   2> 201781 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 201781 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001'
   [junit4]   2> 201781 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 201781 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 201782 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 201782 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 201795 INFO  (zkCallback-409-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6521e82 name:ZooKeeperConnection Watcher:127.0.0.1:45093/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 201795 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 201796 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 201796 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 201797 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001/solr.xml
   [junit4]   2> 201800 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001/cores
   [junit4]   2> 201800 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.CoreContainer New CoreContainer 137008583
   [junit4]   2> 201800 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001]
   [junit4]   2> 201800 WARN  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.CoreContainer Couldn't add files from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001/lib to classpath: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001/lib
   [junit4]   2> 201800 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 201801 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 201801 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 201801 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 201801 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45093/solr
   [junit4]   2> 201801 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 201802 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 201802 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 201805 INFO  (zkCallback-412-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@69803af name:ZooKeeperConnection Watcher:127.0.0.1:45093 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 201806 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 201806 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 201807 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$39/584191008@7ecb6fed
   [junit4]   2> 201807 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 201808 INFO  (zkCallback-413-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3caa6fb name:ZooKeeperConnection Watcher:127.0.0.1:45093/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 201808 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 201808 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 201809 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 201810 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 201811 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 201812 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 201813 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 201813 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 201819 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 201820 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 201820 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 201820 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 201821 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 201821 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 201830 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 201830 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 201831 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96010636938182660-127.0.0.1:36798_-n_0000000000
   [junit4]   2> 201831 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:36798_
   [junit4]   2> 201831 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 201831 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.Overseer Overseer (id=96010636938182660-127.0.0.1:36798_-n_0000000000) starting
   [junit4]   2> 201832 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 201834 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 201834 INFO  (OverseerCollectionConfigSetProcessor-96010636938182660-127.0.0.1:36798_-n_0000000000) [n:127.0.0.1:36798_    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 201834 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36798_
   [junit4]   2> 201834 INFO  (OverseerStateUpdate-96010636938182660-127.0.0.1:36798_-n_0000000000) [n:127.0.0.1:36798_    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 201835 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:36798_
   [junit4]   2> 201838 INFO  (zkCallback-413-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 201838 INFO  (OverseerStateUpdate-96010636938182660-127.0.0.1:36798_-n_0000000000) [n:127.0.0.1:36798_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 201838 INFO  (zkCallback-413-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (1)
   [junit4]   2> 201848 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 201848 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 201848 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001/cores
   [junit4]   2> 201849 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=control_collection, coreNodeName=, dataDir=data/, shard=}
   [junit4]   2> 201849 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.CorePropertiesLocator Found core collection1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001/cores/collection1
   [junit4]   2> 201849 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions
   [junit4]   2> 201849 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.s.SolrDispatchFilter user.dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 201849 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:36798_    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 201849 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 201849 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
   [junit4]   2> 201850 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.ZkController Collection control_collection not visible yet, but flagging it so a watch is registered when it becomes visible
   [junit4]   2> 201850 INFO  (OverseerStateUpdate-96010636938182660-127.0.0.1:36798_-n_0000000000) [n:127.0.0.1:36798_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:36798",
   [junit4]   2>   "node_name":"127.0.0.1:36798_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 201850 INFO  (OverseerStateUpdate-96010636938182660-127.0.0.1:36798_-n_0000000000) [n:127.0.0.1:36798_    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:36798",
   [junit4]   2>   "node_name":"127.0.0.1:36798_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 201853 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.c.ZkStateReader Deleting data for [control_collection]
   [junit4]   2> 201853 INFO  (OverseerStateUpdate-96010636938182660-127.0.0.1:36798_-n_0000000000) [n:127.0.0.1:36798_    ] o.a.s.c.o.ClusterStateMutator building a new cName: control_collection
   [junit4]   2> 201858 INFO  (OverseerStateUpdate-96010636938182660-127.0.0.1:36798_-n_0000000000) [n:127.0.0.1:36798_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 201858 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.ZkController Waiting for coreNodeName for core collection1 in collection control_collection to be assigned
   [junit4]   2> 201859 INFO  (zkCallback-413-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 201859 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection1
   [junit4]   2> 201860 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.ZkController Check for collection zkNode:control_collection
   [junit4]   2> 201860 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.ZkController Collection zkNode exists
   [junit4]   2> 201860 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/control_collection]
   [junit4]   2> 201862 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.c.ZkStateReader path=[/collections/control_collection] [configName]=[conf1] specified config exists in ZooKeeper
   [junit4]   2> 201862 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001/cores/collection1'
   [junit4]   2> 201862 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 201862 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 201863 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.Config loaded config solrconfig.xml with version 0 
   [junit4]   2> 201867 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 201870 WARN  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 201871 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.1.0
   [junit4]   2> 201877 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 201881 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 201944 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.s.IndexSchema default search field in schema is text
   [junit4]   2> 201944 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 201945 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 201946 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 201953 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 201953 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 201953 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore solr.StandardDirectoryFactory
   [junit4]   2> 201957 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001/cores/collection1], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001/cores/collection1/data/]
   [junit4]   2> 201957 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@15f1d171
   [junit4]   2> 201958 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001/cores/collection1/data
   [junit4]   2> 201958 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001/cores/collection1/data/index/
   [junit4]   2> 201958 WARN  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Solr index directory '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 201958 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001/cores/collection1/data/index
   [junit4]   2> 201959 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7249332084837354]
   [junit4]   2> 201969 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001/cores/collection1/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 201969 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 201969 INFO  (OldIndexDirectoryCleanupThreadForCore-collection1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore Looking for old index directories to cleanup for core collection1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001/cores/collection1/data/
   [junit4]   2> 201971 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 201972 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 201972 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 201972 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 201972 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 201972 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 201972 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 201972 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 201972 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 201972 WARN  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,args = {defaults={a=A,b=B}}}
   [junit4]   2> 201975 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 201976 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 201976 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 201977 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 201979 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,/update/json/docs,/admin/luke,/export,/get,/admin/properties,/update/json,/admin/threads,/dump,/update/csv,/sql,/graph,/admin/segments,/admin/system,/replication,/config,/stream,/schema,/admin/plugins,/admin/logging,/admin/ping,/update,/admin/file
   [junit4]   2> 201980 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 201980 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 201980 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 201981 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms; 
   [junit4]   2> 201981 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 201981 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=58.5595703125, floorSegmentMB=1.744140625, forceMergeDeletesPctAllowed=17.68398469445189, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7396171640822374
   [junit4]   2> 201982 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/control-001/cores/collection1/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 201982 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 201982 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@49ea3516[collection1] main]
   [junit4]   2> 201982 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/control_collection]
   [junit4]   2> 201983 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.c.ZkStateReader path=[/collections/control_collection] [configName]=[conf1] specified config exists in ZooKeeper
   [junit4]   2> 201983 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 201983 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 201983 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {}
   [junit4]   2> 201983 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 201983 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 201983 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 201983 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 201983 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 201984 INFO  (searcherExecutor-1162-thread-1-processing-n:127.0.0.1:36798_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@49ea3516[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 201984 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Looking up max value of version field to seed version buckets
   [junit4]   2> 201984 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.VersionInfo Refreshing highest value of _version_ for 65536 version buckets from index
   [junit4]   2> 201984 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.VersionInfo No terms found for _version_, cannot seed version bucket highest value from index
   [junit4]   2> 201984 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1536170191516336128
   [junit4]   2> 201987 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Took 2.0ms to seed version buckets with highest version 1536170191516336128
   [junit4]   2> 201987 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController watch zkdir /configs/conf1
   [junit4]   2> 201987 INFO  (coreLoadExecutor-1161-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.CoreContainer registering core: collection1
   [junit4]   2> 201987 INFO  (coreZkRegister-1155-thread-1-processing-n:127.0.0.1:36798_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController Register replica - core:collection1 address:http://127.0.0.1:36798 collection:control_collection shard:shard1
   [junit4]   2> 201988 INFO  (coreZkRegister-1155-thread-1-processing-n:127.0.0.1:36798_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 201989 INFO  (coreZkRegister-1155-thread-1-processing-n:127.0.0.1:36798_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.LeaderElector Joined leadership election with path: /collections/control_collection/leader_elect/shard1/election/96010636938182660-core_node1-n_0000000000
   [junit4]   2> 201989 INFO  (coreZkRegister-1155-thread-1-processing-n:127.0.0.1:36798_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Running the leader process for shard=shard1 and weAreReplacement=false and leaderVoteWait=10000
   [junit4]   2> 201989 INFO  (coreZkRegister-1155-thread-1-processing-n:127.0.0.1:36798_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 201989 INFO  (coreZkRegister-1155-thread-1-processing-n:127.0.0.1:36798_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 201989 INFO  (coreZkRegister-1155-thread-1-processing-n:127.0.0.1:36798_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:36798/collection1/
   [junit4]   2> 201989 INFO  (coreZkRegister-1155-thread-1-processing-n:127.0.0.1:36798_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 201989 INFO  (coreZkRegister-1155-thread-1-processing-n:127.0.0.1:36798_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:36798/collection1/ has no replicas
   [junit4]   2> 201989 INFO  (OverseerStateUpdate-96010636938182660-127.0.0.1:36798_-n_0000000000) [n:127.0.0.1:36798_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"leader",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection"} current state version: 1
   [junit4]   2> 201990 INFO  (coreZkRegister-1155-thread-1-processing-n:127.0.0.1:36798_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 201991 INFO  (coreZkRegister-1155-thread-1-processing-n:127.0.0.1:36798_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/96010636938182660-core_node1-n_0000000000
   [junit4]   2> 201992 INFO  (coreZkRegister-1155-thread-1-processing-n:127.0.0.1:36798_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:36798/collection1/ shard1
   [junit4]   2> 201992 INFO  (OverseerStateUpdate-96010636938182660-127.0.0.1:36798_-n_0000000000) [n:127.0.0.1:36798_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"leader",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "base_url":"http://127.0.0.1:36798",
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "state":"active"} current state version: 1
   [junit4]   2> 202093 INFO  (zkCallback-413-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 202142 INFO  (coreZkRegister-1155-thread-1-processing-n:127.0.0.1:36798_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController We are http://127.0.0.1:36798/collection1/ and leader is http://127.0.0.1:36798/collection1/
   [junit4]   2> 202142 INFO  (coreZkRegister-1155-thread-1-processing-n:127.0.0.1:36798_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:36798
   [junit4]   2> 202142 INFO  (coreZkRegister-1155-thread-1-processing-n:127.0.0.1:36798_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 202142 INFO  (coreZkRegister-1155-thread-1-processing-n:127.0.0.1:36798_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController publishing state=active
   [junit4]   2> 202142 INFO  (coreZkRegister-1155-thread-1-processing-n:127.0.0.1:36798_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36798_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
   [junit4]   2> 202143 INFO  (OverseerStateUpdate-96010636938182660-127.0.0.1:36798_-n_0000000000) [n:127.0.0.1:36798_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "core_node_name":"core_node1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:36798",
   [junit4]   2>   "node_name":"127.0.0.1:36798_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"} current state version: 2
   [junit4]   2> 202143 INFO  (OverseerStateUpdate-96010636938182660-127.0.0.1:36798_-n_0000000000) [n:127.0.0.1:36798_    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "core_node_name":"core_node1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:36798",
   [junit4]   2>   "node_name":"127.0.0.1:36798_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 202245 INFO  (zkCallback-413-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 202350 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.s.i.CloudSolrClient Final constructed zkHost string: 127.0.0.1:45093/solr
   [junit4]   2> 202351 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 202351 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 202352 INFO  (zkCallback-417-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@29158274 name:ZooKeeperConnection Watcher:127.0.0.1:45093/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 202352 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 202352 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 202352 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 202353 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 202353 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 202354 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 202495 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/shard-1-001/cores/collection1
   [junit4]   2> 202495 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/shard-1-001
   [junit4]   2> 202496 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 202496 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3d1ebca5{/,null,AVAILABLE}
   [junit4]   2> 202497 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.e.j.s.ServerConnector Started ServerConnector@2aee944d{HTTP/1.1,[http/1.1]}{127.0.0.1:35985}
   [junit4]   2> 202497 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.e.j.s.Server Started @204277ms
   [junit4]   2> 202497 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=35985, coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/shard-1-001/cores}
   [junit4]   2> 202498 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 202498 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/shard-1-001'
   [junit4]   2> 202498 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 202498 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 202498 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 202498 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 202499 INFO  (zkCallback-418-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5707dd85 name:ZooKeeperConnection Watcher:127.0.0.1:45093/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 202499 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 202499 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 202500 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 202500 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/shard-1-001/solr.xml
   [junit4]   2> 202504 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/shard-1-001/cores
   [junit4]   2> 202504 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.CoreContainer New CoreContainer 358841709
   [junit4]   2> 202504 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/shard-1-001]
   [junit4]   2> 202504 WARN  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.CoreContainer Couldn't add files from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/shard-1-001/lib to classpath: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/shard-1-001/lib
   [junit4]   2> 202504 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 202505 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 202505 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 202505 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 202505 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45093/solr
   [junit4]   2> 202505 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 202505 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 202505 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 202506 INFO  (zkCallback-421-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5f4d4233 name:ZooKeeperConnection Watcher:127.0.0.1:45093 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 202506 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 202506 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 202507 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$39/584191008@7734b96c
   [junit4]   2> 202507 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 202508 INFO  (zkCallback-422-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3511456e name:ZooKeeperConnection Watcher:127.0.0.1:45093/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 202508 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 202510 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 202510 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 202511 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 202512 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 202512 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96010636938182664-127.0.0.1:35985_-n_0000000001
   [junit4]   2> 202512 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/96010636938182660-127.0.0.1:36798_-n_0000000000 to know if I could be the leader
   [junit4]   2> 202512 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.c.ZkController Publish node=127.0.0.1:35985_ as DOWN
   [junit4]   2> 202513 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35985_
   [junit4]   2> 202514 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:35985_
   [junit4]   2> 202514 INFO  (OverseerStateUpdate-96010636938182660-127.0.0.1:36798_-n_0000000000) [n:127.0.0.1:36798_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"downnode",
   [junit4]   2>   "node_name":"127.0.0.1:35985_"} current state version: 3
   [junit4]   2> 202514 INFO  (OverseerStateUpdate-96010636938182660-127.0.0.1:36798_-n_0000000000) [n:127.0.0.1:36798_    ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:35985_
   [junit4]   2> 202514 INFO  (zkCallback-422-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 202514 INFO  (zkCallback-417-thread-1) [    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 202514 INFO  (zkCallback-413-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 202514 INFO  (zkCallback-413-thread-1-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 202515 INFO  (zkCallback-417-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 202515 INFO  (zkCallback-422-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 202527 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 202527 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 202527 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/shard-1-001/cores
   [junit4]   2> 202528 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=collection1, coreNodeName=, dataDir=data/, shard=}
   [junit4]   2> 202528 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.c.CorePropertiesLocator Found core collection1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/shard-1-001/cores/collection1
   [junit4]   2> 202528 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions
   [junit4]   2> 202528 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 202528 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.s.SolrDispatchFilter user.dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 202528 INFO  (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[CBA5C903E2D068E7]) [n:127.0.0.1:35985_    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 202528 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
   [junit4]   2> 202529 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.ZkController Collection collection1 not visible yet, but flagging it so a watch is registered when it becomes visible
   [junit4]   2> 202529 INFO  (OverseerStateUpdate-96010636938182660-127.0.0.1:36798_-n_0000000000) [n:127.0.0.1:36798_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:35985",
   [junit4]   2>   "node_name":"127.0.0.1:35985_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "operation":"state"} current state version: 3
   [junit4]   2> 202529 INFO  (OverseerStateUpdate-96010636938182660-127.0.0.1:36798_-n_0000000000) [n:127.0.0.1:36798_    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:35985",
   [junit4]   2>   "node_name":"127.0.0.1:35985_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 202529 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.c.ZkStateReader Deleting data for [collection1]
   [junit4]   2> 202529 INFO  (OverseerStateUpdate-96010636938182660-127.0.0.1:36798_-n_0000000000) [n:127.0.0.1:36798_    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
   [junit4]   2> 202529 INFO  (OverseerStateUpdate-96010636938182660-127.0.0.1:36798_-n_0000000000) [n:127.0.0.1:36798_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 202530 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.ZkController Waiting for coreNodeName for core collection1 in collection collection1 to be assigned
   [junit4]   2> 202631 INFO  (zkCallback-417-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [2])
   [junit4]   2> 202631 INFO  (zkCallback-413-thread-2-processing-n:127.0.0.1:36798_) [n:127.0.0.1:36798_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [2])
   [junit4]   2> 202631 INFO  (zkCallback-422-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [2])
   [junit4]   2> 202631 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection1
   [junit4]   2> 202631 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.ZkController Check for collection zkNode:collection1
   [junit4]   2> 202632 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.ZkController Collection zkNode exists
   [junit4]   2> 202632 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/collection1]
   [junit4]   2> 202632 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.c.ZkStateReader path=[/collections/collection1] [configName]=[conf1] specified config exists in ZooKeeper
   [junit4]   2> 202632 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_CBA5C903E2D068E7-001/shard-1-001/cores/collection1'
   [junit4]   2> 202632 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 202632 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 202633 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.Config loaded config solrconfig.xml with version 0 
   [junit4]   2> 202637 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 202639 WARN  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 202640 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.1.0
   [junit4]   2> 202646 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 202649 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 202710 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.s.IndexSchema default search field in schema is text
   [junit4]   2> 202710 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 202711 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 202712 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 202721 INFO  (coreLoadExecutor-1172-thread-1-processing-n:127.0.0.1:35985_) [n:127.0.0.1:35985_ c:collection1   x:collection1] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properti

[...truncated too long message...]

:39179_    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:39179_
   [junit4]   2> 1616901 INFO  (zkCallback-23449-thread-2-processing-n:127.0.0.1:39179_) [n:127.0.0.1:39179_    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1616901 WARN  (zkCallback-23449-thread-2-processing-n:127.0.0.1:39179_) [n:127.0.0.1:39179_    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1616902 INFO  (TEST-DeleteShardTest.test-seed#[CBA5C903E2D068E7]) [    ] o.e.j.s.ServerConnector Stopped ServerConnector@429236e7{SSL,[ssl, http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1616903 INFO  (TEST-DeleteShardTest.test-seed#[CBA5C903E2D068E7]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2e2ece2b{/,null,UNAVAILABLE}
   [junit4]   2> 1616904 INFO  (TEST-DeleteShardTest.test-seed#[CBA5C903E2D068E7]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44918 44918
   [junit4]   2> 1617025 INFO  (Thread-4002) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44918 44918
   [junit4]   2> 1617026 WARN  (Thread-4002) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	4	/solr/aliases.json
   [junit4]   2> 	3	/solr/security.json
   [junit4]   2> 	3	/solr/configs/conf1
   [junit4]   2> 	2	/solr/collections/collection1/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	4	/solr/clusterstate.json
   [junit4]   2> 	4	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	4	/solr/live_nodes
   [junit4]   2> 	4	/solr/collections
   [junit4]   2> 	3	/solr/overseer/queue
   [junit4]   2> 	3	/solr/overseer/collection-queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DeleteShardTest -Dtests.method=test -Dtests.seed=CBA5C903E2D068E7 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ar-OM -Dtests.timezone=Africa/Mbabane -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR    312s J0 | DeleteShardTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.common.SolrException: No registered leader was found after waiting for 10000ms , collection: collection1 slice: shard1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([CBA5C903E2D068E7:43F1F6D94C2C051F]:0)
   [junit4]    > 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:716)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:427)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:332)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:990)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1617029 INFO  (SUITE-DeleteShardTest-seed#[CBA5C903E2D068E7]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteShardTest_CBA5C903E2D068E7-001
   [junit4]   2> Jun 04, 2016 2:31:12 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene60): {}, docValues:{}, maxPointsInLeafNode=1782, maxMBSortInHeap=6.723654992976578, sim=ClassicSimilarity, locale=ar-OM, timezone=Africa/Mbabane
   [junit4]   2> NOTE: Linux 4.4.0-21-generic amd64/Oracle Corporation 1.8.0_92 (64-bit)/cpus=12,threads=1,free=251205664,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [URLClassifyProcessorTest, CacheHeaderTest, HdfsBasicDistributedZk2Test, SynonymTokenizerTest, SparseHLLTest, OutputWriterTest, TestNamedUpdateProcessors, DeleteLastCustomShardedReplicaTest, PathHierarchyTokenizerFactoryTest, ResponseLogComponentTest, TestUseDocValuesAsStored, DateFieldTest, RankQueryTest, TlogReplayBufferedWhileIndexingTest, BlobRepositoryCloudTest, TestCSVLoader, QueryResultKeyTest, HdfsRecoverLeaseTest, WrapperMergePolicyFactoryTest, RAMDirectoryFactoryTest, TestDynamicFieldCollectionResource, SignatureUpdateProcessorFactoryTest, SpatialRPTFieldTypeTest, JSONWriterTest, TestStressLiveNodes, TestManagedSynonymFilterFactory, TestManagedSchema, TestCustomDocTransformer, AlternateDirectoryTest, HdfsSyncSliceTest, DistribJoinFromCollectionTest, SOLR749Test, SharedFSAutoReplicaFailoverTest, TestDistributedSearch, TestSolrConfigHandler, DirectoryFactoryTest, FullHLLTest, TestFieldResource, GraphQueryTest, CollectionTooManyReplicasTest, EchoParamsTest, PolyFieldTest, TestDocBasedVersionConstraints, IndexSchemaRuntimeFieldTest, CdcrReplicationDistributedZkTest, TestBadConfig, TestSortingResponseWriter, ReplicationFactorTest, QueryElevationComponentTest, ConnectionReuseTest, CopyFieldTest, ConjunctionSolrSpellCheckerTest, LeaderInitiatedRecoveryOnCommitTest, TestStandardQParsers, TestJoin, AnalysisAfterCoreReloadTest, TestCrossCoreJoin, ReturnFieldsTest, DistributedDebugComponentTest, TestOmitPositions, TestGraphTermsQParserPlugin, TestSizeLimitedDistributedMap, PrimUtilsTest, TestBinaryField, TestOrdValues, TestNumericTerms64, TestFieldCacheWithThreads, CdcrVersionReplicationTest, CloudExitableDirectoryReaderTest, TestLRUCache, TestConfigSetsAPIExclusivity, TestFieldCache, StandardRequestHandlerTest, CurrencyFieldOpenExchangeTest, HdfsChaosMonkeySafeLeaderTest, ZkNodePropsTest, TestRequestStatusCollectionAPI, TestMiniSolrCloudClusterSSL, TestArbitraryIndexDir, TestDistribDocBasedVersion, TestFaceting, CursorPagingTest, AsyncCallRequestStatusResponseTest, DistanceUnitsTest, TestLazyCores, JavabinLoaderTest, TestFoldingMultitermQuery, ForceLeaderTest, TestExactStatsCache, TestSubQueryTransformerCrossCore, TestJsonRequest, TestRuleBasedAuthorizationPlugin, TestCustomSort, TestDefaultStatsCache, TestSolrQueryParserResource, RequestLoggingTest, DistributedFacetPivotLongTailTest, BasicZkTest, TestReloadDeadlock, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, SyncSliceTest, OpenCloseCoreStressTest, LeaderElectionIntegrationTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, TestRandomFaceting, ZkSolrClientTest, ShardRoutingCustomTest, TestRecovery, TestHashPartitioner, TestStressReorder, TestSolr4Spatial, StatsComponentTest, SolrCmdDistributorTest, BadIndexSchemaTest, TestSort, HighlighterTest, DistributedQueryElevationComponentTest, SuggesterFSTTest, SolrRequestParserTest, SuggesterTSTTest, TestStressLucene, SpatialFilterTest, NoCacheHeaderTest, WordBreakSolrSpellCheckerTest, FieldMutatingUpdateProcessorTest, QueryEqualityTest, DefaultValueUpdateProcessorTest, StatelessScriptUpdateProcessorFactoryTest, IndexBasedSpellCheckerTest, LukeRequestHandlerTest, DocumentAnalysisRequestHandlerTest, FileBasedSpellCheckerTest, TestValueSourceCache, TermVectorComponentTest, RequiredFieldsTest, LoggingHandlerTest, TestAnalyzedSuggestions, TestPartialUpdateDeduplication, CSVRequestHandlerTest, TestComponentsName, HighlighterConfigTest, TestQuerySenderListener, UpdateParamsTest, TestSolrIndexConfig, TestQuerySenderNoQuery, TestMergePolicyConfig, MultiTermTest, TestDocSet, NumericFieldsTest, TestElisionMultitermQuery, TestConfig, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, TestSolrCoreProperties, NotRequiredUniqueKeyTest, TestLuceneMatchVersion, TestPhraseSuggestions, SpellPossibilityIteratorTest, TestCharFilters, TestXIncludeConfig, TestSweetSpotSimilarityFactory, TestDFRSimilarityFactory, TestLMDirichletSimilarityFactory, TestIBSimilarityFactory, TestBM25SimilarityFactory, ResourceLoaderTest, TestFastOutputStream, OpenExchangeRatesOrgProviderTest, PluginInfoTest, TestFastLRUCache, PreAnalyzedFieldTest, TestSuggestSpellingConverter, SpellingQueryConverterTest, ClusterStateTest, SliceStateTest, DistributedMLTComponentTest, AliasIntegrationTest, BaseCdcrDistributedZkTest, CollectionReloadTest, CollectionsAPIAsyncDistributedZkTest, DeleteInactiveReplicaTest, DeleteShardTest]
   [junit4] Completed [468/613 (2!)] on J0 in 329.09s, 2 tests, 1 error <<< FAILURES!

[...truncated 459 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 1 error, 1 failure, 116 ignored (72 assumptions) [seed: CBA5C903E2D068E7]

Total time: 54 minutes 23 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-6.x-Linux (64bit/jdk1.8.0_92) - Build # 840 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/840/
Java: 64bit/jdk1.8.0_92 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

3 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestCoreDiscovery

Error Message:
ObjectTracker found 4 object(s) that were not released!!! [MockDirectoryWrapper, MDCAwareThreadPoolExecutor, SolrCore, MockDirectoryWrapper]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 4 object(s) that were not released!!! [MockDirectoryWrapper, MDCAwareThreadPoolExecutor, SolrCore, MockDirectoryWrapper]
	at __randomizedtesting.SeedInfo.seed([AE7EF232A10F4145]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor32.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestCoreDiscovery

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.core.TestCoreDiscovery:     1) Thread[id=8920, name=searcherExecutor-4505-thread-1, state=WAITING, group=TGRP-TestCoreDiscovery]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestCoreDiscovery: 
   1) Thread[id=8920, name=searcherExecutor-4505-thread-1, state=WAITING, group=TGRP-TestCoreDiscovery]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([AE7EF232A10F4145]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestCoreDiscovery

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=8920, name=searcherExecutor-4505-thread-1, state=WAITING, group=TGRP-TestCoreDiscovery]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=8920, name=searcherExecutor-4505-thread-1, state=WAITING, group=TGRP-TestCoreDiscovery]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([AE7EF232A10F4145]:0)




Build Log:
[...truncated 11968 lines...]
   [junit4] Suite: org.apache.solr.core.TestCoreDiscovery
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/init-core-data-001
   [junit4]   2> 1244388 INFO  (SUITE-TestCoreDiscovery-seed#[AE7EF232A10F4145]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1244388 INFO  (SUITE-TestCoreDiscovery-seed#[AE7EF232A10F4145]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 1244388 INFO  (SUITE-TestCoreDiscovery-seed#[AE7EF232A10F4145]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 1244389 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testAlternateCoreDir
   [junit4]   2> 1244392 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244393 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-001
   [junit4]   2> 1244393 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-001'
   [junit4]   2> 1244393 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244393 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-001
   [junit4]   2> 1244393 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-001/solr.xml
   [junit4]   2> 1244396 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002
   [junit4]   2> 1244396 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CoreContainer New CoreContainer 1385185418
   [junit4]   2> 1244396 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-001]
   [junit4]   2> 1244396 WARN  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CoreContainer Couldn't add files from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-001/lib to classpath: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-001/lib
   [junit4]   2> 1244396 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 600000,connTimeout : 60000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 1244398 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
   [junit4]   2> 1244399 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrCores Allocating transient cache for 2 transient cores
   [junit4]   2> 1244400 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1244400 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1244400 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 1244400 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 1244401 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002
   [junit4]   2> 1244401 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=core2, config=solrconfig-minimal.xml, loadOnStartup=false, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=false, dataDir=core2}
   [junit4]   2> 1244402 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CorePropertiesLocator Found core core2 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2
   [junit4]   2> 1244402 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=core1, config=solrconfig-minimal.xml, loadOnStartup=true, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=false, dataDir=core1}
   [junit4]   2> 1244402 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CorePropertiesLocator Found core core1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1
   [junit4]   2> 1244402 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CorePropertiesLocator Found 2 core definitions
   [junit4]   2> 1244402 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1'
   [junit4]   2> 1244402 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244402 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-001
   [junit4]   2> 1244408 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 1244410 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.1.0
   [junit4]   2> 1244426 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig-minimal.xml
   [junit4]   2> 1244429 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.s.IndexSchema [core1] Schema name=tiny
   [junit4]   2> 1244432 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 1244436 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1'
   [junit4]   2> 1244437 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.c.CoreContainer Creating SolrCore 'core1' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1
   [junit4]   2> 1244437 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1244437 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrCore [[core1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1/]
   [junit4]   2> 1244437 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrCore JMX monitoring not detected for core: core1
   [junit4]   2> 1244437 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1
   [junit4]   2> 1244437 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrCore New index directory detected: old=null new=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1/index/
   [junit4]   2> 1244438 WARN  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrCore [core1] Solr index directory '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1/index' doesn't exist. Creating new index...
   [junit4]   2> 1244438 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1/index
   [junit4]   2> 1244438 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=38.2763671875, floorSegmentMB=2.154296875, forceMergeDeletesPctAllowed=1.3293771542077448, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1244438 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@381be64c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@27e00987),segFN=segments_1,generation=1}
   [junit4]   2> 1244438 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244439 INFO  (OldIndexDirectoryCleanupThreadForCore-core1) [    x:core1] o.a.s.c.SolrCore Looking for old index directories to cleanup for core core1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1/
   [junit4]   2> 1244439 WARN  (OldIndexDirectoryCleanupThreadForCore-core1) [    x:core1] o.a.s.c.DirectoryFactory /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1/ does not point to a valid data directory; skipping clean-up of old index directories.
   [junit4]   2> 1244441 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1244443 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244443 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244444 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244444 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244445 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,/update/csv,/sql,/update/json/docs,/admin/luke,/graph,/export,/select,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/stream,/schema,/admin/plugins,/admin/logging,/update/json,/admin/ping,/admin/threads,/update,/admin/file
   [junit4]   2> 1244445 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1244447 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1244447 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1244447 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=42, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6196029480165364]
   [junit4]   2> 1244447 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@381be64c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@27e00987),segFN=segments_1,generation=1}
   [junit4]   2> 1244447 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244448 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.s.SolrIndexSearcher Opening [Searcher@42a4a269[core1] main]
   [junit4]   2> 1244448 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/conf
   [junit4]   2> 1244448 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/conf}
   [junit4]   2> 1244448 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using file:dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/conf
   [junit4]   2> 1244448 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 1244448 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1244448 INFO  (searcherExecutor-4442-thread-1-processing-x:core1) [    x:core1] o.a.s.c.SolrCore [core1] Registered new searcher Searcher@42a4a269[core1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1244448 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.CoreContainer registering core: core1
   [junit4]   2> 1244448 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2'
   [junit4]   2> 1244449 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244449 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-001
   [junit4]   2> 1244454 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 1244457 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.1.0
   [junit4]   2> 1244462 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig-minimal.xml
   [junit4]   2> 1244466 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.s.IndexSchema [core2] Schema name=tiny
   [junit4]   2> 1244468 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 1244469 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2'
   [junit4]   2> 1244469 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CoreContainer Creating SolrCore 'core2' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2
   [junit4]   2> 1244469 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1244469 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrCore [[core2] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2/]
   [junit4]   2> 1244469 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrCore JMX monitoring not detected for core: core2
   [junit4]   2> 1244469 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2
   [junit4]   2> 1244469 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrCore New index directory detected: old=null new=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2/index/
   [junit4]   2> 1244470 WARN  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrCore [core2] Solr index directory '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2/index' doesn't exist. Creating new index...
   [junit4]   2> 1244470 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2/index
   [junit4]   2> 1244470 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=21, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1244470 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@2ce41f44 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@36bd269c),segFN=segments_1,generation=1}
   [junit4]   2> 1244470 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244470 INFO  (OldIndexDirectoryCleanupThreadForCore-core2) [    x:core2] o.a.s.c.SolrCore Looking for old index directories to cleanup for core core2 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2/
   [junit4]   2> 1244471 WARN  (OldIndexDirectoryCleanupThreadForCore-core2) [    x:core2] o.a.s.c.DirectoryFactory /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2/ does not point to a valid data directory; skipping clean-up of old index directories.
   [junit4]   2> 1244472 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1244473 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244473 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244473 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244474 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244475 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,/update/csv,/sql,/update/json/docs,/admin/luke,/graph,/export,/select,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/stream,/schema,/admin/plugins,/admin/logging,/update/json,/admin/ping,/admin/threads,/update,/admin/file
   [junit4]   2> 1244475 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1244475 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1244476 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1244476 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=77.130859375, floorSegmentMB=0.486328125, forceMergeDeletesPctAllowed=29.512037338187863, segmentsPerTier=34.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1244476 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@2ce41f44 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@36bd269c),segFN=segments_1,generation=1}
   [junit4]   2> 1244476 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244476 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.s.SolrIndexSearcher Opening [Searcher@37c84115[core2] main]
   [junit4]   2> 1244477 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/conf
   [junit4]   2> 1244477 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/conf}
   [junit4]   2> 1244477 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using file:dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/conf
   [junit4]   2> 1244477 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 1244477 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1244477 INFO  (searcherExecutor-4446-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrCore [core2] Registered new searcher Searcher@37c84115[core2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1244477 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer registering core: core2
   [junit4]   2> 1244478 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1385185418
   [junit4]   2> 1244478 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.c.SolrCore [core2]  CLOSING SolrCore org.apache.solr.core.SolrCore@33e9c1e2
   [junit4]   2> 1244478 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.u.DirectUpdateHandler2 closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 1244478 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.u.SolrCoreState Closing SolrCoreState
   [junit4]   2> 1244478 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.u.DefaultSolrCoreState SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1244478 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.u.DefaultSolrCoreState closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1244479 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.c.SolrCore [core2] Closing main searcher on request.
   [junit4]   2> 1244479 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.c.CachingDirectoryFactory Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1244479 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.c.CachingDirectoryFactory looking to close /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2 [CachedDir<<refCount=0;path=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2;done=false>>]
   [junit4]   2> 1244479 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 1244479 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.c.CachingDirectoryFactory Closing directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2
   [junit4]   2> 1244479 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.c.CachingDirectoryFactory looking to close /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2/index [CachedDir<<refCount=0;path=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2/index;done=false>>]
   [junit4]   2> 1244479 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 1244479 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.c.CachingDirectoryFactory Closing directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2/index
   [junit4]   2> 1244494 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.c.SolrCore [core1]  CLOSING SolrCore org.apache.solr.core.SolrCore@440e8e7f
   [junit4]   2> 1244494 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.u.DirectUpdateHandler2 closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 1244494 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.u.SolrCoreState Closing SolrCoreState
   [junit4]   2> 1244494 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.u.DefaultSolrCoreState SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1244494 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.u.DefaultSolrCoreState closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1244495 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.c.SolrCore [core1] Closing main searcher on request.
   [junit4]   2> 1244495 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.c.CachingDirectoryFactory Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1244495 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.c.CachingDirectoryFactory looking to close /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1 [CachedDir<<refCount=0;path=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1;done=false>>]
   [junit4]   2> 1244495 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 1244495 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.c.CachingDirectoryFactory Closing directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1
   [junit4]   2> 1244495 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.c.CachingDirectoryFactory looking to close /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1/index [CachedDir<<refCount=0;path=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1/index;done=false>>]
   [junit4]   2> 1244495 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 1244495 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.c.CachingDirectoryFactory Closing directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1/index
   [junit4]   2> 1244495 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.SolrTestCaseJ4 ###Ending testAlternateCoreDir
   [junit4]   2> 1244497 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.SolrTestCaseJ4 ###Starting testDuplicateNames
   [junit4]   2> 1244500 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244501 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003
   [junit4]   2> 1244501 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003'
   [junit4]   2> 1244501 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244501 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003
   [junit4]   2> 1244501 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003/solr.xml
   [junit4]   2> 1244503 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003
   [junit4]   2> 1244503 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer New CoreContainer 511314936
   [junit4]   2> 1244503 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003]
   [junit4]   2> 1244504 WARN  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer Couldn't add files from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003/lib to classpath: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003/lib
   [junit4]   2> 1244504 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 600000,connTimeout : 60000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 1244504 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
   [junit4]   2> 1244505 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrCores Allocating transient cache for 2 transient cores
   [junit4]   2> 1244505 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1244505 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1244505 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 1244505 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 1244505 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003
   [junit4]   2> 1244505 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=core1, config=solrconfig-minimal.xml, loadOnStartup=false, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=false, dataDir=stuffandnonsense}
   [junit4]   2> 1244506 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found core core1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003/core2
   [junit4]   2> 1244506 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=core1, config=solrconfig-minimal.xml, loadOnStartup=true, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=false, dataDir=stuffandnonsense}
   [junit4]   2> 1244506 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found core core1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003/core1
   [junit4]   2> 1244506 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found 2 core definitions
   [junit4]   2> 1244506 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer Shutting down CoreContainer instance=511314936
   [junit4]   2> 1244506 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.SolrTestCaseJ4 ###Ending testDuplicateNames
   [junit4]   2> 1244508 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.SolrTestCaseJ4 ###Starting testTooManyTransientCores
   [junit4]   2> 1244516 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004'
   [junit4]   2> 1244516 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244516 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004
   [junit4]   2> 1244516 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/solr.xml
   [junit4]   2> 1244519 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004
   [junit4]   2> 1244519 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer New CoreContainer 243489109
   [junit4]   2> 1244519 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004]
   [junit4]   2> 1244519 WARN  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer Couldn't add files from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/lib to classpath: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/lib
   [junit4]   2> 1244519 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 600000,connTimeout : 60000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 1244520 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
   [junit4]   2> 1244521 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrCores Allocating transient cache for 2 transient cores
   [junit4]   2> 1244521 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1244521 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1244521 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 1244521 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 1244521 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004
   [junit4]   2> 1244522 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=coreT5, config=solrconfig-minimal.xml, loadOnStartup=true, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=true, dataDir=coreT5}
   [junit4]   2> 1244522 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found core coreT5 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5
   [junit4]   2> 1244522 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=coreT1, config=solrconfig-minimal.xml, loadOnStartup=true, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=true, dataDir=coreT1}
   [junit4]   2> 1244522 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found core coreT1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1
   [junit4]   2> 1244522 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=coreLOS, config=solrconfig-minimal.xml, loadOnStartup=true, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=false, dataDir=coreLOS}
   [junit4]   2> 1244522 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found core coreLOS in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS
   [junit4]   2> 1244523 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=coreT4, config=solrconfig-minimal.xml, loadOnStartup=true, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=true, dataDir=coreT4}
   [junit4]   2> 1244523 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found core coreT4 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT4
   [junit4]   2> 1244523 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=coreT2, config=solrconfig-minimal.xml, loadOnStartup=true, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=true, dataDir=coreT2}
   [junit4]   2> 1244523 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found core coreT2 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT2
   [junit4]   2> 1244524 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=coreT6, config=solrconfig-minimal.xml, loadOnStartup=true, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=true, dataDir=coreT6}
   [junit4]   2> 1244524 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found core coreT6 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT6
   [junit4]   2> 1244524 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=coreT3, config=solrconfig-minimal.xml, loadOnStartup=true, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=true, dataDir=coreT3}
   [junit4]   2> 1244524 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found core coreT3 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT3
   [junit4]   2> 1244524 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found 7 core definitions
   [junit4]   2> 1244525 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5'
   [junit4]   2> 1244525 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244525 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004
   [junit4]   2> 1244529 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1'
   [junit4]   2> 1244529 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS'
   [junit4]   2> 1244529 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244529 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004
   [junit4]   2> 1244529 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244529 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004
   [junit4]   2> 1244533 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 1244533 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 1244534 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 1244534 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.1.0
   [junit4]   2> 1244549 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.1.0
   [junit4]   2> 1244552 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig-minimal.xml
   [junit4]   2> 1244554 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.s.IndexSchema [coreT1] Schema name=tiny
   [junit4]   2> 1244556 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 1244557 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1'
   [junit4]   2> 1244557 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.c.CoreContainer Creating SolrCore 'coreT1' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1
   [junit4]   2> 1244557 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1244558 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrCore [[coreT1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/coreT1/]
   [junit4]   2> 1244558 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrCore JMX monitoring not detected for core: coreT1
   [junit4]   2> 1244558 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/coreT1
   [junit4]   2> 1244558 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.1.0
   [junit4]   2> 1244562 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig-minimal.xml
   [junit4]   2> 1244563 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig-minimal.xml
   [junit4]   2> 1244564 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrCore New index directory detected: old=null new=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/coreT1/index/
   [junit4]   2> 1244564 WARN  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrCore [coreT1] Solr index directory '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/coreT1/index' doesn't exist. Creating new index...
   [junit4]   2> 1244564 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/coreT1/index
   [junit4]   2> 1244564 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=38.2763671875, floorSegmentMB=2.154296875, forceMergeDeletesPctAllowed=1.3293771542077448, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1244565 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@532e2ba lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@26edd96f),segFN=segments_1,generation=1}
   [junit4]   2> 1244565 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244565 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.s.IndexSchema [coreT5] Schema name=tiny
   [junit4]   2> 1244565 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.s.IndexSchema [coreLOS] Schema name=tiny
   [junit4]   2> 1244567 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 1244567 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 1244567 INFO  (OldIndexDirectoryCleanupThreadForCore-coreT1) [    x:coreT1] o.a.s.c.SolrCore Looking for old index directories to cleanup for core coreT1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/coreT1/
   [junit4]   2> 1244567 WARN  (OldIndexDirectoryCleanupThreadForCore-coreT1) [    x:coreT1] o.a.s.c.DirectoryFactory /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/coreT1/ does not point to a valid data directory; skipping clean-up of old index directories.
   [junit4]   2> 1244568 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5'
   [junit4]   2> 1244568 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.CoreContainer Creating SolrCore 'coreT5' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5
   [junit4]   2> 1244568 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS'
   [junit4]   2> 1244568 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.c.CoreContainer Creating SolrCore 'coreLOS' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS
   [junit4]   2> 1244568 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1244568 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1244568 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrCore [[coreT5] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/coreT5/]
   [junit4]   2> 1244568 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrCore [[coreLOS] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/coreLOS/]
   [junit4]   2> 1244568 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrCore JMX monitoring not detected for core: coreT5
   [junit4]   2> 1244569 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrCore JMX monitoring not detected for core: coreLOS
   [junit4]   2> 1244569 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/coreT5
   [junit4]   2> 1244569 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/coreLOS
   [junit4]   2> 1244569 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrCore New index directory detected: old=null new=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/coreT5/index/
   [junit4]   2> 1244569 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrCore New index directory detected: old=null new=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/coreLOS/index/
   [junit4]   2> 1244569 WARN  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrCore [coreT5] Solr index directory '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/coreT5/index' doesn't exist. Creating new index...
   [junit4]   2> 1244569 WARN  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrCore [coreLOS] Solr index directory '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/coreLOS/index' doesn't exist. Creating new index...
   [junit4]   2> 1244569 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/coreT5/index
   [junit4]   2> 1244570 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/coreLOS/index
   [junit4]   2> 1244570 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=38.2763671875, floorSegmentMB=2.154296875, forceMergeDeletesPctAllowed=1.3293771542077448, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1244570 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=38.2763671875, floorSegmentMB=2.154296875, forceMergeDeletesPctAllowed=1.3293771542077448, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1244570 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@6071c9f3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2e2fcf84),segFN=segments_1,generation=1}
   [junit4]   2> 1244570 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244570 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@7e01285f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@e5eced1),segFN=segments_1,generation=1}
   [junit4]   2> 1244570 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244570 INFO  (OldIndexDirectoryCleanupThreadForCore-coreT5) [    x:coreT5] o.a.s.c.SolrCore Looking for old index directories to cleanup for core coreT5 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/coreT5/
   [junit4]   2> 1244570 WARN  (OldIndexDirectoryCleanupThreadForCore-coreT5) [    x:coreT5] o.a.s.c.DirectoryFactory /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/coreT5/ does not point to a valid data directory; skipping clean-up of old index directories.
   [junit4]   2> 1244571 INFO  (OldIndexDirectoryCleanupThreadForCore-coreLOS) [    x:coreLOS] o.a.s.c.SolrCore Looking for old index directories to cleanup for core coreLOS in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/coreLOS/
   [junit4]   2> 1244571 WARN  (OldIndexDirectoryCleanupThreadForCore-coreLOS) [    x:coreLOS] o.a.s.c.DirectoryFactory /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/coreLOS/ does not point to a valid data directory; skipping clean-up of old index directories.
   [junit4]   2> 1244572 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1244572 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1244572 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1244573 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244573 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244573 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244574 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244574 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244574 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244574 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244575 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244575 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244575 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,/update/csv,/sql,/update/json/docs,/admin/luke,/graph,/export,/select,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/stream,/schema,/admin/plugins,/admin/logging,/update/json,/admin/ping,/admin/threads,/update,/admin/file
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=42, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6196029480165364]
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@6071c9f3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2e2fcf84),segFN=segments_1,generation=1}
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.s.SolrIndexSearcher Opening [Searcher@6a3f5db8[coreT5] main]
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/conf
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/conf}
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,/update/csv,/sql,/update/json/docs,/admin/luke,/graph,/export,/select,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/stream,/schema,/admin/plugins,/admin/logging,/update/json,/admin/ping,/admin/threads,/update,/admin/file
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using file:dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/conf
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,/update/csv,/sql,/update/json/docs,/admin/luke,/graph,/export,/select,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/stream,/schema,/admin/plugins,/admin/logging,/update/json,/admin/ping,/admin/threads,/update,/admin/file
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1244577 INFO  (searcherExecutor-4466-thread-1-processing-x:coreT5) [    x:coreT5] o.a.s.c.SolrCore [coreT5] Registered new searcher Searcher@6a3f5db8[coreT5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=42, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6196029480165364]
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrCores Opening transient core coreT5
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.CoreContainer registering core: coreT5
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@532e2ba lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@26edd96f),segFN=segments_1,generation=1}
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT4'
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=42, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6196029480165364]
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@7e01285f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@e5eced1),segFN=segments_1,generation=1}
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.s.SolrIndexSearcher Opening [Searcher@154d1d1c[coreT1] main]
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/conf
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.s.SolrIndexSearcher Opening [Searcher@58a9e044[coreLOS] main]
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/conf}
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using file:dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/conf
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/conf
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/conf}
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using file:dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/conf
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1244579 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 1244579 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1244579 INFO  (searcherExecutor-4465-thread-1-processing-x:coreT1) [    x:coreT1] o.a.s.c.SolrCore [coreT1] Registered new searcher Searcher@154d1d1c[coreT1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1244579 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrCores Opening transient core coreT1
   [junit4]   2> 1244579 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.CoreContainer registering core: coreT1
   [junit4]   2> 1244579 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.CoreContainer registering core: coreLOS
   [junit4]   2> 1244579 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspa

[...truncated too long message...]

Thread.java:745)
   [junit4]   2> 
   [junit4]   2> Jun 07, 2016 2:17:23 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> Jun 07, 2016 2:17:43 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.solr.core.TestCoreDiscovery: 
   [junit4]   2>    1) Thread[id=8920, name=searcherExecutor-4505-thread-1, state=WAITING, group=TGRP-TestCoreDiscovery]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Jun 07, 2016 2:17:43 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=8920, name=searcherExecutor-4505-thread-1, state=WAITING, group=TGRP-TestCoreDiscovery]
   [junit4]   2> Jun 07, 2016 2:17:46 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=8920, name=searcherExecutor-4505-thread-1, state=WAITING, group=TGRP-TestCoreDiscovery]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Lucene60, sim=ClassicSimilarity, locale=et, timezone=America/Regina
   [junit4]   2> NOTE: Linux 4.4.0-21-generic amd64/Oracle Corporation 1.8.0_92 (64-bit)/cpus=12,threads=2,free=116467560,total=518148096
   [junit4]   2> NOTE: All tests run in this JVM: [NoCacheHeaderTest, TestSolrQueryParser, TestCryptoKeys, PreAnalyzedUpdateProcessorTest, TestPerFieldSimilarity, ZkStateWriterTest, CleanupOldIndexTest, TestConfigSetImmutable, MergeStrategyTest, TestExactSharedStatsCache, TestInitParams, TestSolrIndexConfig, BJQParserTest, HdfsNNFailoverTest, TestFuzzyAnalyzedSuggestions, StatelessScriptUpdateProcessorFactoryTest, HighlighterConfigTest, HardAutoCommitTest, ShowFileRequestHandlerTest, TestSolrXml, SuggesterTest, TestReload, TestScoreJoinQPScore, TestRestoreCore, TestDistributedMissingSort, ResponseHeaderTest, TestShardHandlerFactory, TestFieldTypeResource, SegmentsInfoRequestHandlerTest, CdcrRequestHandlerTest, TestPKIAuthenticationPlugin, TestSolrDeletionPolicy1, TermVectorComponentDistributedTest, DirectSolrConnectionTest, BasicAuthIntegrationTest, PrimitiveFieldTypeTest, BlockJoinFacetSimpleTest, BigEndianAscendingWordDeserializerTest, LukeRequestHandlerTest, TestCollationField, TestFastWriter, ShardSplitTest, LeaderElectionTest, TestCloudInspectUtil, TestSchemaSimilarityResource, AtomicUpdatesTest, TestLockTree, TestRandomCollapseQParserPlugin, TestStressCloudBlindAtomicUpdates, XmlUpdateRequestHandlerTest, TestSweetSpotSimilarityFactory, SpatialHeatmapFacetsTest, OverseerStatusTest, TestRandomRequestDistribution, TestTolerantSearch, TestRecovery, DistributedFacetPivotLargeTest, TestSolrQueryParserDefaultOperatorResource, TestZkChroot, TestSort, StatsComponentTest, HdfsChaosMonkeyNothingIsSafeTest, TestConfigSetsAPI, AssignTest, BadComponentTest, TestIndexingPerformance, DistributedFacetPivotSmallTest, RemoteQueryErrorTest, TestPostingsSolrHighlighter, TestReversedWildcardFilterFactory, TestHdfsUpdateLog, TestIndexSearcher, TestRTimerTree, TestWriterPerf, RegexBoostProcessorTest, MigrateRouteKeyTest, QueryParsingTest, TestComplexPhraseQParserPlugin, TestJmxMonitoredMap, RecoveryZkTest, TestRequestForwarding, UUIDUpdateProcessorFallbackTest, PluginInfoTest, SimpleCollectionCreateDeleteTest, HLLUtilTest, TestSolrQueryResponse, HdfsTlogReplayBufferedWhileIndexingTest, TestNumericTerms32, TestGroupingSearch, TestBulkSchemaAPI, EnumFieldTest, DistributedVersionInfoTest, TestTolerantUpdateProcessorRandomCloud, DistribDocExpirationUpdateProcessorTest, DebugComponentTest, RequestHandlersTest, BasicZkTest, FullSolrCloudDistribCmdsTest, TestDistributedSearch, DistributedSpellCheckComponentTest, TestStressVersions, SimpleFacetsTest, TestSolr4Spatial, SolrCmdDistributorTest, TestFunctionQuery, BasicFunctionalityTest, DirectUpdateHandlerTest, TestBadConfig, SoftAutoCommitTest, SimplePostToolTest, TestCoreDiscovery]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestCoreDiscovery -Dtests.seed=AE7EF232A10F4145 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=et -Dtests.timezone=America/Regina -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J2 | TestCoreDiscovery (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ObjectTracker found 4 object(s) that were not released!!! [MockDirectoryWrapper, MDCAwareThreadPoolExecutor, SolrCore, MockDirectoryWrapper]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AE7EF232A10F4145]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestCoreDiscovery: 
   [junit4]    >    1) Thread[id=8920, name=searcherExecutor-4505-thread-1, state=WAITING, group=TGRP-TestCoreDiscovery]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AE7EF232A10F4145]:0)Throwable #3: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=8920, name=searcherExecutor-4505-thread-1, state=WAITING, group=TGRP-TestCoreDiscovery]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AE7EF232A10F4145]:0)
   [junit4] Completed [446/613 (1!)] on J2 in 145.29s, 12 tests, 1 failure, 2 errors <<< FAILURES!

[...truncated 551 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 3 suite-level errors, 682 ignored (67 assumptions) [seed: AE7EF232A10F4145]

Total time: 48 minutes 4 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS-EA] Lucene-Solr-6.x-Linux (64bit/jdk-9-ea+121) - Build # 839 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/839/
Java: 64bit/jdk-9-ea+121 -XX:+UseCompressedOops -XX:+UseParallelGC

17 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
Error from server at http://127.0.0.1:45528/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:45528/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([B721922F7358ADDD:3F75ADF5DDA4C025]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.tryDelete(ChaosMonkeySafeLeaderTest.java:190)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:92)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.SimpleCollectionCreateDeleteTest.test

Error Message:
No registered leader was found after waiting for 10000ms , collection: collection1 slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 10000ms , collection: collection1 slice: shard1
	at __randomizedtesting.SeedInfo.seed([B721922F7358ADDD:3F75ADF5DDA4C025]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:718)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:427)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:332)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:990)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
17 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest:     1) Thread[id=1081, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[B721922F7358ADDD]-SendThread(127.0.0.1:42177), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    2) Thread[id=1060, name=Thread-389, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    3) Thread[id=1054, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[B721922F7358ADDD]-SendThread(127.0.0.1:34628), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    4) Thread[id=1082, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[B721922F7358ADDD]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    5) Thread[id=1086, name=OverseerHdfsCoreFailoverThread-96026543810281475-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    6) Thread[id=1059, name=OverseerHdfsCoreFailoverThread-96026522827489283-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=1091, name=zkCallback-183-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    8) Thread[id=1057, name=OverseerStateUpdate-96026522827489283-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=1085, name=OverseerCollectionConfigSetProcessor-96026543810281475-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   10) Thread[id=1058, name=OverseerCollectionConfigSetProcessor-96026522827489283-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   11) Thread[id=1087, name=Thread-391, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)   12) Thread[id=1093, name=zkCallback-183-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   13) Thread[id=1055, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[B721922F7358ADDD]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   14) Thread[id=1084, name=OverseerStateUpdate-96026543810281475-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)         at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   15) Thread[id=1089, name=zkCallback-183-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   16) Thread[id=1090, name=zkCallback-183-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   17) Thread[id=1092, name=zkCallback-183-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 17 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest: 
   1) Thread[id=1081, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[B721922F7358ADDD]-SendThread(127.0.0.1:42177), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   2) Thread[id=1060, name=Thread-389, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   3) Thread[id=1054, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[B721922F7358ADDD]-SendThread(127.0.0.1:34628), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   4) Thread[id=1082, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[B721922F7358ADDD]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   5) Thread[id=1086, name=OverseerHdfsCoreFailoverThread-96026543810281475-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   6) Thread[id=1059, name=OverseerHdfsCoreFailoverThread-96026522827489283-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   7) Thread[id=1091, name=zkCallback-183-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   8) Thread[id=1057, name=OverseerStateUpdate-96026522827489283-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=1085, name=OverseerCollectionConfigSetProcessor-96026543810281475-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  10) Thread[id=1058, name=OverseerCollectionConfigSetProcessor-96026522827489283-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  11) Thread[id=1087, name=Thread-391, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
  12) Thread[id=1093, name=zkCallback-183-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  13) Thread[id=1055, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[B721922F7358ADDD]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  14) Thread[id=1084, name=OverseerStateUpdate-96026543810281475-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
        at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)
        at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)
        at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  15) Thread[id=1089, name=zkCallback-183-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  16) Thread[id=1090, name=zkCallback-183-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  17) Thread[id=1092, name=zkCallback-183-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
	at __randomizedtesting.SeedInfo.seed([B721922F7358ADDD]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=1081, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[B721922F7358ADDD]-SendThread(127.0.0.1:42177), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    2) Thread[id=1087, name=Thread-391, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    3) Thread[id=1093, name=zkCallback-183-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    4) Thread[id=1060, name=Thread-389, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    5) Thread[id=1054, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[B721922F7358ADDD]-SendThread(127.0.0.1:34628), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    6) Thread[id=1086, name=OverseerHdfsCoreFailoverThread-96026543810281475-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=1059, name=OverseerHdfsCoreFailoverThread-96026522827489283-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    8) Thread[id=1091, name=zkCallback-183-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=1089, name=zkCallback-183-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   10) Thread[id=1090, name=zkCallback-183-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   11) Thread[id=1092, name=zkCallback-183-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=1081, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[B721922F7358ADDD]-SendThread(127.0.0.1:42177), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   2) Thread[id=1087, name=Thread-391, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   3) Thread[id=1093, name=zkCallback-183-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   4) Thread[id=1060, name=Thread-389, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   5) Thread[id=1054, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[B721922F7358ADDD]-SendThread(127.0.0.1:34628), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   6) Thread[id=1086, name=OverseerHdfsCoreFailoverThread-96026543810281475-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   7) Thread[id=1059, name=OverseerHdfsCoreFailoverThread-96026522827489283-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   8) Thread[id=1091, name=zkCallback-183-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=1089, name=zkCallback-183-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  10) Thread[id=1090, name=zkCallback-183-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  11) Thread[id=1092, name=zkCallback-183-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
	at __randomizedtesting.SeedInfo.seed([B721922F7358ADDD]:0)


FAILED:  org.apache.solr.cloud.AliasIntegrationTest.test

Error Message:
Error from server at https://127.0.0.1:42951/lee/v/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /lee/v/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:42951/lee/v/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /lee/v/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([B721922F7358ADDD:3F75ADF5DDA4C025]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.AliasIntegrationTest.test(AliasIntegrationTest.java:69)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.AsyncMigrateRouteKeyTest.test

Error Message:
Error from server at http://127.0.0.1:33488/at_jgc/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /at_jgc/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:33488/at_jgc/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /at_jgc/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([B721922F7358ADDD:3F75ADF5DDA4C025]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest(MigrateRouteKeyTest.java:135)
	at org.apache.solr.cloud.AsyncMigrateRouteKeyTest.test(AsyncMigrateRouteKeyTest.java:44)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
Error from server at http://127.0.0.1:34084: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore

Stack Trace:
java.lang.AssertionError: Error from server at http://127.0.0.1:34084: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore
	at __randomizedtesting.SeedInfo.seed([B721922F7358ADDD:3F75ADF5DDA4C025]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:162)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:71)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Error from server at http://127.0.0.1:34798/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolr

[...truncated too long message...]

rDispatchFilter.java:208)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
   [junit4]    > 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]    > 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
   [junit4]    > 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]    > 	at org.eclipse.jetty.server.Server.handle(Server.java:518)
   [junit4]    > 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
   [junit4]    > 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
   [junit4]    > 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]    > 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]    > 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]    > 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
   [junit4]    > 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    > Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
   [junit4]    > 	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
   [junit4]    > 	... 1 more
   [junit4]    > ,code=500}</pre></p>
   [junit4]    > <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
   [junit4]    > </body>
   [junit4]    > </html>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B721922F7358ADDD:3F75ADF5DDA4C025]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
   [junit4]    > 	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:128)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]   2> 10414172 INFO  (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[B721922F7358ADDD]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_B721922F7358ADDD-001
   [junit4]   2> Jun 06, 2016 11:54:29 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene60): {}, docValues:{}, maxPointsInLeafNode=242, maxMBSortInHeap=5.517363650119625, sim=RandomSimilarity(queryNorm=false,coord=crazy): {}, locale=lu, timezone=Antarctica/Casey
   [junit4]   2> NOTE: Linux 4.4.0-21-generic amd64/Oracle Corporation 9-ea (64-bit)/cpus=12,threads=1,free=160036344,total=455606272
   [junit4]   2> NOTE: All tests run in this JVM: [TestQuerySenderListener, TestStressUserVersions, ChaosMonkeySafeLeaderTest, DistanceFunctionTest, TestScoreJoinQPNoScore, TestConfigSets, TestLegacyFieldCache, FileUtilsTest, TestQueryUtils, TestPivotHelperCode, TestSolr4Spatial2, TestUniqueKeyFieldResource, CollectionsAPIDistributedZkTest, TestChildDocTransformer, TestMiniSolrCloudClusterBase, SpellPossibilityIteratorTest, DirectSolrSpellCheckerTest, DeleteReplicaTest, HdfsWriteToMultipleCollectionsTest, ExitableDirectoryReaderTest, TestFieldTypeCollectionResource, DistributedQueueTest, TestTrie, OpenCloseCoreStressTest, SolrCmdDistributorTest, HighlighterMaxOffsetTest, TestSha256AuthenticationProvider, SharedFSAutoReplicaFailoverUtilsTest, TestLRUStatsCache, TestGeoJSONResponseWriter, HdfsRecoveryZkTest, UniqFieldsUpdateProcessorFactoryTest, HdfsBasicDistributedZkTest, TestAnalyzeInfixSuggestions, TestDistribIDF, AsyncMigrateRouteKeyTest, TestPartialUpdateDeduplication, TestSolrCoreProperties, MultiTermTest, TestFieldCacheReopen, TestDistributedGrouping, OutOfBoxZkACLAndCredentialsProvidersTest, TestCoreContainer, ShardRoutingCustomTest, CSVRequestHandlerTest, TestStressVersions, TestTestInjection, TestUseDocValuesAsStored2, SpellCheckComponentTest, AliasIntegrationTest, TestSubQueryTransformerDistrib, TestFiltering, TolerantUpdateProcessorTest, ZkStateReaderTest, DistributedIntervalFacetingTest, RestartWhileUpdatingTest, SolrXmlInZkTest, NotRequiredUniqueKeyTest, TestFreeTextSuggestions, TestTolerantUpdateProcessorCloud, SuggesterFSTTest, TestCollectionAPI, DistributedExpandComponentTest, SecurityConfHandlerTest, HttpPartitionTest, HLLSerializationTest, TestBlobHandler, TestConfigOverlay, CollectionReloadTest, SolrInfoMBeanTest, TestSurroundQueryParser, DistributedMLTComponentTest, TestManagedSchemaAPI, TestDynamicFieldResource, BlockDirectoryTest, DistributedQueryComponentCustomSortTest, TestFieldSortValues, RecoveryAfterSoftCommitTest, TestDownShardTolerantSearch, ShardRoutingTest, TestSolrDeletionPolicy2, TestCursorMarkWithoutUniqueKey, BadCopyFieldTest, DOMUtilTest, BasicFunctionalityTest, TestRestManager, TestCloudDeleteByQuery, TestReplicaProperties, TestSolrConfigHandlerCloud, TestInfoStreamLogging, HdfsCollectionsAPIDistributedZkTest, TestQuerySenderNoQuery, ChaosMonkeyNothingIsSafeTest]
   [junit4] Completed [607/613 (14!)] on J1 in 612.35s, 1 test, 1 error <<< FAILURES!

[...truncated 38 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 2 suite-level errors, 11 errors, 4 failures, 1777 ignored (63 assumptions) [seed: B721922F7358ADDD]

Total time: 197 minutes 14 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS-EA] Lucene-Solr-6.x-Linux (64bit/jdk-9-ea+121) - Build # 838 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/838/
Java: 64bit/jdk-9-ea+121 -XX:-UseCompressedOops -XX:+UseG1GC

22 tests failed.
FAILED:  org.apache.solr.schema.TestCloudManagedSchema.test

Error Message:
Schema resource name differs from expected name expected:<[managed-schema]> but was:<[schema.xml]>

Stack Trace:
org.junit.ComparisonFailure: Schema resource name differs from expected name expected:<[managed-schema]> but was:<[schema.xml]>
	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE:51D4AD8E435BFC36]:0)
	at org.junit.Assert.assertEquals(Assert.java:125)
	at org.apache.solr.schema.TestCloudManagedSchema.test(TestCloudManagedSchema.java:68)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.CollectionTooManyReplicasTest.testAddTooManyReplicas

Error Message:
No registered leader was found after waiting for 10000ms , collection: collection1 slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 10000ms , collection: collection1 slice: shard1
	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE:54D2E68CA341AE7F]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:718)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:427)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:332)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:990)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
15 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest:     1) Thread[id=7095, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[D9809254EDA791CE]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    2) Thread[id=7074, name=Thread-2371, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    3) Thread[id=7094, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:39710), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    4) Thread[id=7103, name=zkCallback-1274-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    5) Thread[id=7099, name=OverseerHdfsCoreFailoverThread-96025796035018755-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    6) Thread[id=7072, name=OverseerCollectionConfigSetProcessor-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=7102, name=zkCallback-1274-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    8) Thread[id=7098, name=OverseerCollectionConfigSetProcessor-96025796035018755-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=7069, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[D9809254EDA791CE]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   10) Thread[id=7104, name=zkCallback-1274-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   11) Thread[id=7097, name=OverseerStateUpdate-96025796035018755-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)         at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   12) Thread[id=7100, name=Thread-2373, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)   13) Thread[id=7071, name=OverseerStateUpdate-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   14) Thread[id=7073, name=OverseerHdfsCoreFailoverThread-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   15) Thread[id=7068, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:35172), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest: 
   1) Thread[id=7095, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[D9809254EDA791CE]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   2) Thread[id=7074, name=Thread-2371, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   3) Thread[id=7094, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:39710), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   4) Thread[id=7103, name=zkCallback-1274-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   5) Thread[id=7099, name=OverseerHdfsCoreFailoverThread-96025796035018755-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   6) Thread[id=7072, name=OverseerCollectionConfigSetProcessor-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   7) Thread[id=7102, name=zkCallback-1274-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   8) Thread[id=7098, name=OverseerCollectionConfigSetProcessor-96025796035018755-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=7069, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[D9809254EDA791CE]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  10) Thread[id=7104, name=zkCallback-1274-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  11) Thread[id=7097, name=OverseerStateUpdate-96025796035018755-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
        at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)
        at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)
        at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  12) Thread[id=7100, name=Thread-2373, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
  13) Thread[id=7071, name=OverseerStateUpdate-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  14) Thread[id=7073, name=OverseerHdfsCoreFailoverThread-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  15) Thread[id=7068, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:35172), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=7102, name=zkCallback-1274-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    2) Thread[id=7074, name=Thread-2371, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    3) Thread[id=7094, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:39710), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    4) Thread[id=7103, name=zkCallback-1274-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    5) Thread[id=7099, name=OverseerHdfsCoreFailoverThread-96025796035018755-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    6) Thread[id=7104, name=zkCallback-1274-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=7100, name=Thread-2373, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    8) Thread[id=7073, name=OverseerHdfsCoreFailoverThread-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=7068, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:35172), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=7102, name=zkCallback-1274-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   2) Thread[id=7074, name=Thread-2371, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   3) Thread[id=7094, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:39710), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   4) Thread[id=7103, name=zkCallback-1274-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   5) Thread[id=7099, name=OverseerHdfsCoreFailoverThread-96025796035018755-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   6) Thread[id=7104, name=zkCallback-1274-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   7) Thread[id=7100, name=Thread-2373, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   8) Thread[id=7073, name=OverseerHdfsCoreFailoverThread-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=7068, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:35172), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE]:0)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
Error from server at http://127.0.0.1:35328/n_/p: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore

Stack Trace:
java.lang.AssertionError: Error from server at http://127.0.0.1:35328/n_/p: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore
	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE:51D4AD8E435BFC36]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:162)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:71)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:44310/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44310/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE:51D4AD8E435BFC36]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:152)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicZkTest.testBasic

Error Message:
No registered leader was found after waiting for 30000ms , collection: collection1 slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 30000ms , collection: collection1 slice: shard1
	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE:727A8F41327B17E0]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:718)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:685)
	at org.apache.solr.cloud.BasicZkTest.testBasic(BasicZkTest.java:51)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Error from server at http://127.0.0.1:45997/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:45997/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE:51D4AD8E435BFC36]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:128)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration

Error Message:
Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: Timed out getting shard id for core: testcore

Stack Trace:
org.apache.solr.common.SolrException: Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: Timed out getting shard id for core: testcore
	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE:670BF4FB94DD9FFB]:0)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:785)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:718)
	at org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration(ClusterStateUpdateTest.java:158)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Unable to create core [testcore]
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:828)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:751)
	... 41 more
Caused by: org.apache.solr.common.SolrException: Timed out getting shard id for core: testcore
	at org.apache.solr.cloud.ZkController.waitForShardId(ZkController.java:1407)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1362)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	... 42 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:46176

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:46176
	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE:51D4AD8E435BFC36]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:601)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:399)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testNoConfigSetExist(CollectionsAPIDistributedZkTest.java:601)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:176)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(java.base@9-ea/Native Method)
	at java.net.SocketInputStream.socketRead(java.base@9-ea/SocketInputStream.java:116)
	at java.net.SocketInputStream.read(java.base@9-ea/SocketInputStream.java:170)
	at java.net.SocketInputStream.read(java.base@9-ea/SocketInputStream.java:141)
	at sun.security.ssl.SSLSocketInputRecord.read(java.base@9-ea/SSLSocketInputRecord.java:425)
	at sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(java.base@9-ea/SSLSocketInputRecord.java:65)
	at sun.security.ssl.SSLSocketImpl.bytesInCompletePacket(java.base@9-ea/SSLSocketImpl.java:911)
	at sun.security.ssl.AppInputStream.read(java.base@9-ea/AppInputStream.java:144)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:495)
	... 4

[...truncated too long message...]

rStateUpdate-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
   [junit4]    >         at java.lang.Thread.sleep(java.base@9-ea/Native Method)
   [junit4]    >         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
   [junit4]    >         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
   [junit4]    >         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
   [junit4]    >         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)
   [junit4]    >         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
   [junit4]    >         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    >   14) Thread[id=7073, name=OverseerHdfsCoreFailoverThread-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
   [junit4]    >         at java.lang.Thread.sleep(java.base@9-ea/Native Method)
   [junit4]    >         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
   [junit4]    >         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    >   15) Thread[id=7068, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:35172), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
   [junit4]    >         at java.lang.Thread.sleep(java.base@9-ea/Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=7102, name=zkCallback-1274-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
   [junit4]    >         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
   [junit4]    >         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    >    2) Thread[id=7074, name=Thread-2371, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
   [junit4]    >         at java.lang.Object.wait(java.base@9-ea/Native Method)
   [junit4]    >         at java.lang.Object.wait(java.base@9-ea/Object.java:516)
   [junit4]    >         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   [junit4]    >    3) Thread[id=7094, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:39710), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
   [junit4]    >         at java.lang.Thread.sleep(java.base@9-ea/Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    >    4) Thread[id=7103, name=zkCallback-1274-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
   [junit4]    >         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
   [junit4]    >         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    >    5) Thread[id=7099, name=OverseerHdfsCoreFailoverThread-96025796035018755-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
   [junit4]    >         at java.lang.Thread.sleep(java.base@9-ea/Native Method)
   [junit4]    >         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
   [junit4]    >         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    >    6) Thread[id=7104, name=zkCallback-1274-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
   [junit4]    >         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
   [junit4]    >         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    >    7) Thread[id=7100, name=Thread-2373, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
   [junit4]    >         at java.lang.Object.wait(java.base@9-ea/Native Method)
   [junit4]    >         at java.lang.Object.wait(java.base@9-ea/Object.java:516)
   [junit4]    >         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   [junit4]    >    8) Thread[id=7073, name=OverseerHdfsCoreFailoverThread-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
   [junit4]    >         at java.lang.Thread.sleep(java.base@9-ea/Native Method)
   [junit4]    >         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
   [junit4]    >         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    >    9) Thread[id=7068, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:35172), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
   [junit4]    >         at java.lang.Thread.sleep(java.base@9-ea/Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE]:0)
   [junit4] Completed [317/613 (19!)] on J1 in 665.76s, 2 tests, 2 failures, 2 errors <<< FAILURES!

[...truncated 999 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 2 suite-level errors, 15 errors, 5 failures, 1110 ignored (72 assumptions) [seed: D9809254EDA791CE]

Total time: 183 minutes 55 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS-EA] Lucene-Solr-6.x-Linux (64bit/jdk-9-ea+121) - Build # 837 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/837/
Java: 64bit/jdk-9-ea+121 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

20 tests failed.
FAILED:  org.apache.solr.schema.TestBulkSchemaConcurrent.test

Error Message:
[[], [], [], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"]], []]

Stack Trace:
java.lang.AssertionError: [[], [], [], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"]], []]
	at __randomizedtesting.SeedInfo.seed([13E501B446D4C3CA:9BB13E6EE828AE32]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.schema.TestBulkSchemaConcurrent.test(TestBulkSchemaConcurrent.java:115)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest:     1) Thread[id=7019, name=Thread-2827, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    2) Thread[id=7045, name=Thread-2829, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    3) Thread[id=7049, name=zkCallback-1123-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    4) Thread[id=7047, name=zkCallback-1123-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    5) Thread[id=7050, name=zkCallback-1123-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    6) Thread[id=7016, name=OverseerStateUpdate-96024570223591427-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=7042, name=OverseerStateUpdate-96024591208284163-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)         at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    8) Thread[id=7044, name=OverseerHdfsCoreFailoverThread-96024591208284163-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=7018, name=OverseerHdfsCoreFailoverThread-96024570223591427-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   10) Thread[id=7017, name=OverseerCollectionConfigSetProcessor-96024570223591427-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   11) Thread[id=7013, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[13E501B446D4C3CA]-SendThread(127.0.0.1:43486), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)   12) Thread[id=7039, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[13E501B446D4C3CA]-SendThread(127.0.0.1:36307), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)   13) Thread[id=7048, name=zkCallback-1123-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   14) Thread[id=7043, name=OverseerCollectionConfigSetProcessor-96024591208284163-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   15) Thread[id=7040, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[13E501B446D4C3CA]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   16) Thread[id=7014, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[13E501B446D4C3CA]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest: 
   1) Thread[id=7019, name=Thread-2827, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   2) Thread[id=7045, name=Thread-2829, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   3) Thread[id=7049, name=zkCallback-1123-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   4) Thread[id=7047, name=zkCallback-1123-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   5) Thread[id=7050, name=zkCallback-1123-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   6) Thread[id=7016, name=OverseerStateUpdate-96024570223591427-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   7) Thread[id=7042, name=OverseerStateUpdate-96024591208284163-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
        at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)
        at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)
        at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   8) Thread[id=7044, name=OverseerHdfsCoreFailoverThread-96024591208284163-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=7018, name=OverseerHdfsCoreFailoverThread-96024570223591427-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  10) Thread[id=7017, name=OverseerCollectionConfigSetProcessor-96024570223591427-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  11) Thread[id=7013, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[13E501B446D4C3CA]-SendThread(127.0.0.1:43486), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
  12) Thread[id=7039, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[13E501B446D4C3CA]-SendThread(127.0.0.1:36307), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
  13) Thread[id=7048, name=zkCallback-1123-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  14) Thread[id=7043, name=OverseerCollectionConfigSetProcessor-96024591208284163-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  15) Thread[id=7040, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[13E501B446D4C3CA]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  16) Thread[id=7014, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[13E501B446D4C3CA]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
	at __randomizedtesting.SeedInfo.seed([13E501B446D4C3CA]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=7047, name=zkCallback-1123-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    2) Thread[id=7050, name=zkCallback-1123-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    3) Thread[id=7044, name=OverseerHdfsCoreFailoverThread-96024591208284163-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    4) Thread[id=7018, name=OverseerHdfsCoreFailoverThread-96024570223591427-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    5) Thread[id=7013, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[13E501B446D4C3CA]-SendThread(127.0.0.1:43486), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    6) Thread[id=7019, name=Thread-2827, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    7) Thread[id=7039, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[13E501B446D4C3CA]-SendThread(127.0.0.1:36307), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    8) Thread[id=7048, name=zkCallback-1123-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=7045, name=Thread-2829, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)   10) Thread[id=7049, name=zkCallback-1123-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=7047, name=zkCallback-1123-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   2) Thread[id=7050, name=zkCallback-1123-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   3) Thread[id=7044, name=OverseerHdfsCoreFailoverThread-96024591208284163-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   4) Thread[id=7018, name=OverseerHdfsCoreFailoverThread-96024570223591427-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   5) Thread[id=7013, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[13E501B446D4C3CA]-SendThread(127.0.0.1:43486), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   6) Thread[id=7019, name=Thread-2827, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   7) Thread[id=7039, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[13E501B446D4C3CA]-SendThread(127.0.0.1:36307), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   8) Thread[id=7048, name=zkCallback-1123-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=7045, name=Thread-2829, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
  10) Thread[id=7049, name=zkCallback-1123-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
	at __randomizedtesting.SeedInfo.seed([13E501B446D4C3CA]:0)


FAILED:  org.apache.solr.cloud.AliasIntegrationTest.test

Error Message:
Error from server at https://127.0.0.1:40154/_dwu/ey/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /_dwu/ey/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:40154/_dwu/ey/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /_dwu/ey/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([13E501B446D4C3CA:9BB13E6EE828AE32]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.AliasIntegrationTest.test(AliasIntegrationTest.java:69)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.AsyncMigrateRouteKeyTest.test

Error Message:
Error from server at http://127.0.0.1:38584/qgkz/o/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /qgkz/o/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:38584/qgkz/o/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /qgkz/o/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([13E501B446D4C3CA:9BB13E6EE828AE32]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest(MigrateRouteKeyTest.java:135)
	at org.apache.solr.cloud.AsyncMigrateRouteKeyTest.test(AsyncMigrateRouteKeyTest.java:44)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
Error from server at http://127.0.0.1:38695: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore

Stack Trace:
java.lang.AssertionError: Error from server at http://127.0.0.1:38695: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore
	at __randomizedtesting.SeedInfo.seed([13E501B446D4C3CA:9BB13E6EE828AE32]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:162)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:71)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:33368/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:33368/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([13E501B446D4C3CA:9BB13E6EE828AE32]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:152)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicZkTest.testBasic

Error Message:
No registered leader was found after waiting for 30000ms , collection: collection1 slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 30000ms , collection: collection1 slice: shard1
	at __randomizedtesting.SeedInfo.seed([13E501B446D4C3CA:B81F1CA1990845E4]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:718)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:685)
	at org.apache.solr.cloud.BasicZkTest.testBasic(BasicZkTest.java:51)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.te

[...truncated too long message...]

e.CoreContainer.getCore(CoreContainer.java:1080)
   [junit4]    > 	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
   [junit4]    > 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
   [junit4]    > 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]    > 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
   [junit4]    > 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]    > 	at org.eclipse.jetty.server.Server.handle(Server.java:518)
   [junit4]    > 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
   [junit4]    > 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
   [junit4]    > 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]    > 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]    > 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)
   [junit4]    > 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]    > 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]    > 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]    > 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
   [junit4]    > 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    > Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
   [junit4]    > 	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
   [junit4]    > 	... 1 more
   [junit4]    > ,code=500}</pre></p>
   [junit4]    > <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
   [junit4]    > </body>
   [junit4]    > </html>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([13E501B446D4C3CA:9BB13E6EE828AE32]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
   [junit4]    > 	at org.apache.solr.cloud.RemoteQueryErrorTest.test(RemoteQueryErrorTest.java:49)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]   2> 8677219 INFO  (SUITE-RemoteQueryErrorTest-seed#[13E501B446D4C3CA]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.RemoteQueryErrorTest_13E501B446D4C3CA-001
   [junit4]   2> Jun 06, 2016 1:38:01 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene60): {}, docValues:{}, maxPointsInLeafNode=1897, maxMBSortInHeap=7.52727781057156, sim=RandomSimilarity(queryNorm=false,coord=yes): {}, locale=de-AT, timezone=Pacific/Tongatapu
   [junit4]   2> NOTE: Linux 4.4.0-21-generic amd64/Oracle Corporation 9-ea (64-bit)/cpus=12,threads=1,free=81611880,total=508887040
   [junit4]   2> NOTE: All tests run in this JVM: [RulesTest, DocValuesMultiTest, DeleteShardTest, TestDistributedMissingSort, TestSolrIndexConfig, HdfsTlogReplayBufferedWhileIndexingTest, DirectUpdateHandlerOptimizeTest, AnalyticsMergeStrategyTest, TestElisionMultitermQuery, SolrTestCaseJ4Test, TestGroupingSearch, TestExactSharedStatsCache, MergeStrategyTest, CleanupOldIndexTest, SimpleFacetsTest, DistribDocExpirationUpdateProcessorTest, DistributedFacetPivotSmallTest, DocExpirationUpdateProcessorFactoryTest, SolrCoreCheckLockOnStartupTest, CollectionStateFormat2Test, XmlUpdateRequestHandlerTest, TermVectorComponentDistributedTest, TestFieldTypeResource, OverseerTest, FullSolrCloudDistribCmdsTest, DirectUpdateHandlerTest, TestHdfsUpdateLog, TestCloudManagedSchema, BigEndianAscendingWordDeserializerTest, TestSolrXml, DirectSolrConnectionTest, TestRestoreCore, TestPKIAuthenticationPlugin, BlockJoinFacetSimpleTest, TestJettySolrRunner, BadIndexSchemaTest, RegexBoostProcessorTest, PreAnalyzedUpdateProcessorTest, UpdateParamsTest, TestPerFieldSimilarity, DistributedQueryComponentOptimizationTest, TestConfigReload, TestSolrFieldCacheMBean, TestFilteredDocIdSet, CurrencyFieldXmlFileTest, TestSolrConfigHandlerConcurrent, TestNumericTerms32, TestShardHandlerFactory, TestCSVResponseWriter, TestCoreDiscovery, StatsComponentTest, TestTolerantUpdateProcessorRandomCloud, HardAutoCommitTest, TestDynamicLoading, ZkStateWriterTest, RollingRestartTest, TestReversedWildcardFilterFactory, TestPHPSerializedResponseWriter, DataDrivenBlockJoinTest, PluginInfoTest, TestTolerantSearch, DateMathParserTest, TestExclusionRuleCollectionAccess, TestLockTree, RemoteQueryErrorTest]
   [junit4] Completed [578/613 (17!)] on J2 in 615.95s, 1 test, 1 error <<< FAILURES!

[...truncated 152 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 2 suite-level errors, 14 errors, 4 failures, 1150 ignored (68 assumptions) [seed: 13E501B446D4C3CA]

Total time: 181 minutes 31 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-6.x-Linux (64bit/jdk1.8.0_92) - Build # 836 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/836/
Java: 64bit/jdk1.8.0_92 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

12 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
Doc with id=2 not found in http://127.0.0.1:35483/qkaxn/c8n_1x2_leader_session_loss due to: Path not found: /id; rsp={doc=null}

Stack Trace:
java.lang.AssertionError: Doc with id=2 not found in http://127.0.0.1:35483/qkaxn/c8n_1x2_leader_session_loss due to: Path not found: /id; rsp={doc=null}
	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:F79042921BB9E834]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.HttpPartitionTest.assertDocExists(HttpPartitionTest.java:604)
	at org.apache.solr.cloud.HttpPartitionTest.assertDocsExistInAllReplicas(HttpPartitionTest.java:559)
	at org.apache.solr.cloud.HttpPartitionTest.testLeaderZkSessionLoss(HttpPartitionTest.java:507)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:120)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest:     1) Thread[id=1098, name=zkCallback-211-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=1092, name=OverseerCollectionConfigSetProcessor-96022541866565635-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=1093, name=OverseerHdfsCoreFailoverThread-96022541866565635-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=1094, name=Thread-381, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    5) Thread[id=1066, name=OverseerHdfsCoreFailoverThread-96022520875778051-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=1062, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[7FC47D48B54585CC]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    7) Thread[id=1088, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[7FC47D48B54585CC]-SendThread(127.0.0.1:46786), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    8) Thread[id=1097, name=zkCallback-211-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=1061, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[7FC47D48B54585CC]-SendThread(127.0.0.1:44336), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   10) Thread[id=1099, name=zkCallback-211-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   11) Thread[id=1089, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[7FC47D48B54585CC]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   12) Thread[id=1064, name=OverseerStateUpdate-96022520875778051-127.0.0.1:7000_solr-n_0000000000, state=WAITING, group=Overseer state updater.]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342)         at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153)         at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348)         at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)         at java.lang.Thread.run(Thread.java:745)   13) Thread[id=1067, name=Thread-379, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)   14) Thread[id=1096, name=zkCallback-211-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   15) Thread[id=1065, name=OverseerCollectionConfigSetProcessor-96022520875778051-127.0.0.1:7000_solr-n_0000000000, state=WAITING, group=Overseer collection creation process.]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342)         at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153)         at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348)         at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(Thread.java:745)   16) Thread[id=1091, name=OverseerStateUpdate-96022541866565635-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)         at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest: 
   1) Thread[id=1098, name=zkCallback-211-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=1092, name=OverseerCollectionConfigSetProcessor-96022541866565635-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=1093, name=OverseerHdfsCoreFailoverThread-96022541866565635-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=1094, name=Thread-381, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   5) Thread[id=1066, name=OverseerHdfsCoreFailoverThread-96022520875778051-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=1062, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[7FC47D48B54585CC]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   7) Thread[id=1088, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[7FC47D48B54585CC]-SendThread(127.0.0.1:46786), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   8) Thread[id=1097, name=zkCallback-211-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=1061, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[7FC47D48B54585CC]-SendThread(127.0.0.1:44336), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  10) Thread[id=1099, name=zkCallback-211-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  11) Thread[id=1089, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[7FC47D48B54585CC]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  12) Thread[id=1064, name=OverseerStateUpdate-96022520875778051-127.0.0.1:7000_solr-n_0000000000, state=WAITING, group=Overseer state updater.]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
        at java.lang.Thread.run(Thread.java:745)
  13) Thread[id=1067, name=Thread-379, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
  14) Thread[id=1096, name=zkCallback-211-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  15) Thread[id=1065, name=OverseerCollectionConfigSetProcessor-96022520875778051-127.0.0.1:7000_solr-n_0000000000, state=WAITING, group=Overseer collection creation process.]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(Thread.java:745)
  16) Thread[id=1091, name=OverseerStateUpdate-96022541866565635-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
        at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)
        at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)
        at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=1098, name=zkCallback-211-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=1093, name=OverseerHdfsCoreFailoverThread-96022541866565635-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=1094, name=Thread-381, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    4) Thread[id=1066, name=OverseerHdfsCoreFailoverThread-96022520875778051-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=1088, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[7FC47D48B54585CC]-SendThread(127.0.0.1:46786), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    6) Thread[id=1097, name=zkCallback-211-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=1067, name=Thread-379, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    8) Thread[id=1061, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[7FC47D48B54585CC]-SendThread(127.0.0.1:44336), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    9) Thread[id=1096, name=zkCallback-211-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=1099, name=zkCallback-211-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=1098, name=zkCallback-211-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=1093, name=OverseerHdfsCoreFailoverThread-96022541866565635-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=1094, name=Thread-381, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   4) Thread[id=1066, name=OverseerHdfsCoreFailoverThread-96022520875778051-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=1088, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[7FC47D48B54585CC]-SendThread(127.0.0.1:46786), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   6) Thread[id=1097, name=zkCallback-211-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=1067, name=Thread-379, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   8) Thread[id=1061, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[7FC47D48B54585CC]-SendThread(127.0.0.1:44336), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   9) Thread[id=1096, name=zkCallback-211-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=1099, name=zkCallback-211-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC]:0)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
Error from server at http://127.0.0.1:39972: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore

Stack Trace:
java.lang.AssertionError: Error from server at http://127.0.0.1:39972: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore
	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:F79042921BB9E834]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:162)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:71)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Error from server at http://127.0.0.1:36216/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:36216/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:F79042921BB9E834]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:128)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration

Error Message:
Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: Timed out getting shard id for core: testcore

Stack Trace:
org.apache.solr.common.SolrException: Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: Timed out getting shard id for core: testcore
	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:C14F1BE7CC3F8BF9]:0)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:785)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:718)
	at org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration(ClusterStateUpdateTest.java:158)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: Unable to create core [testcore]
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:828)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:751)
	... 41 more
Caused by: org.apache.solr.common.SolrException: Timed out getting shard id for core: testcore
	at org.apache.solr.cloud.ZkController.waitForShardId(ZkController.java:1407)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1362)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	... 42 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:42963

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:42963
	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:F79042921BB9E834]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:601)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:399)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testNoConfigSetExist(CollectionsAPIDistributedZkTest.java:601)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:176)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:495)
	... 47 more


FAILED:  org.apache.solr.cloud.DistribCursorPagingTest.test

Error Message:
Error from server at http://127.0.0.1:35284/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:35284/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:F79042921BB9E834]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.DistribCursorPagingTest.doBadInputTest(DistribCursorPagingTest.java:114)
	at org.apache.solr.cloud.DistribCursorPagingTest.test(DistribCursorPagingTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout

Error Message:
Could not start container:7000. Reason:class org.apache.solr.common.SolrException Error CREATEing SolrCore 'collection1_shard1': Unable to create core [collection1_shard1] Caused by: Timed out getting coreNodeName for collection1_shard1

Stack Trace:
java.lang.AssertionError: Could not start container:7000. Reason:class org.apache.solr.common.SolrException Error CREATEing SolrCore 'collection1_shard1': Unable to create core [collection1_shard1] Caused by: Timed out getting coreNodeName for collection1_shard1
	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:D6F3D8EB92600D27]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.LeaderElectionIntegrationTest.setUp(LeaderElectionIntegrationTest.java:102)
	at sun.reflect.GeneratedMethodAccessor18.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:905)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderElectionIntegrationTest.testSimpleSliceLeaderElection

Error Message:
Could not start container:7000. Reason:class org.apache.solr.common.SolrException Error CREATEing SolrCore 'collection1_shard1': Unable to create core [collection1_shard1] Caused by: Timed out getting coreNodeName for collection1_shard1

Stack Trace:
java.lang.AssertionError: Could not start container:7000. Reason:class org.apache.solr.common.SolrException Error CREATEing SolrCore 'collection1_shard1': Unable to create core [collection1_shard1] Caused by: Timed out getting coreNodeName for collection1_shard1
	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:21F0B006F63A192B]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.LeaderElectionIntegrationTest.setUp(LeaderElectionIntegrationTest.java:102)
	at sun.reflect.GeneratedMethodAccessor18.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:905)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:


Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:F79042921BB9E834]:0)
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertTrue(Assert.java:54)
	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:127)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.ShardRoutingTest.test

Error Message:
Error from server at https://127.0.0.1:35969/rw_zzh/co/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /rw_zzh/co/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:35969/rw_zzh/co/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /rw_zzh/co/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:F79042921BB9E834]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.indexDoc(AbstractFullDistribZkTestBase.java:775)
	at org.apache.solr.BaseDistributedSearchTestCase.index(BaseDistributedSearchTestCase.java:483)
	at org.apache.solr.cloud.ShardRoutingTest.doAddDoc(ShardRoutingTest.java:324)
	at org.apache.solr.cloud.ShardRoutingTest.doHashingTest(ShardRoutingTest.java:132)
	at org.apache.solr.cloud.ShardRoutingTest.test(ShardRoutingTest.java:109)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10691 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_7FC47D48B54585CC-001/init-core-data-001
   [junit4]   2> 0    INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[7FC47D48B54585CC]-worker) [    ] o.e.j.u.log Logging initialized @1720ms
   [junit4]   2> 6    INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[7FC47D48B54585CC]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 14   INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[7FC47D48B54585CC]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 38   INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[7FC47D48B54585CC]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 40   INFO  (Thread-1) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 40   INFO  (Thread-1) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 140  INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[7FC47D48B54585CC]) [    ] o.a.s.c.ZkTestServer start zk server on port:35617
   [junit4]   2> 161  INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[7FC47D48B54585CC]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 192  INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[7FC47D48B54585CC]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 245  INFO  (zkCallback-1-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@77e66d38 name:ZooKeeperConnection Watcher:127.0.0.1:35617 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 245  INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[7FC47D48B54585CC]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 247  INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[7FC47D48B54585CC]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 249  INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[7FC47D48B54585CC]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 277  WARN  (NIOServerCxn.Factory

[...truncated too long message...]

127.0.0.1:1663_solr    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 5290364 ERROR (OverseerHdfsCoreFailoverThread-96022820617125895-127.0.0.1:1663_solr-n_0000000002) [n:127.0.0.1:1663_solr    ] o.a.s.c.OverseerAutoReplicaFailoverThread OverseerAutoReplicaFailoverThread had an error in its thread work loop.:org.apache.solr.common.SolrException: Could not load collection from ZK: collection1
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1050)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:610)
   [junit4]   2> 	at org.apache.solr.common.cloud.ClusterState.getCollectionsMap(ClusterState.java:248)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.doWork(OverseerAutoReplicaFailoverThread.java:164)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:129)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.InterruptedException
   [junit4]   2> 	at java.lang.Object.wait(Native Method)
   [junit4]   2> 	at java.lang.Object.wait(Object.java:502)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1059)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1045)
   [junit4]   2> 	... 5 more
   [junit4]   2> 
   [junit4]   2> 5290364 WARN  (zkCallback-1020-thread-2-processing-n:127.0.0.1:1663_solr) [n:127.0.0.1:1663_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 5290366 INFO  (TEST-ClusterStateUpdateTest.testCoreRegistration-seed#[7FC47D48B54585CC]) [n:127.0.0.1:1661_solr    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38874 38874
   [junit4]   2> 5290384 INFO  (Thread-2164) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38874 38874
   [junit4]   2> 5290385 WARN  (Thread-2164) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	3	/solr/security.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	3	/solr/collections
   [junit4]   2> 	2	/solr/overseer/collection-queue-work
   [junit4]   2> 
   [junit4]   2> 5290385 INFO  (TEST-ClusterStateUpdateTest.testCoreRegistration-seed#[7FC47D48B54585CC]) [n:127.0.0.1:1661_solr    ] o.a.s.SolrTestCaseJ4 ###Ending testCoreRegistration
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ClusterStateUpdateTest -Dtests.method=testCoreRegistration -Dtests.seed=7FC47D48B54585CC -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=hi -Dtests.timezone=Asia/Hebron -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR    322s J1 | ClusterStateUpdateTest.testCoreRegistration <<<
   [junit4]    > Throwable #1: org.apache.solr.common.SolrException: Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: Timed out getting shard id for core: testcore
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:C14F1BE7CC3F8BF9]:0)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:785)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:718)
   [junit4]    > 	at org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration(ClusterStateUpdateTest.java:158)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.common.SolrException: Unable to create core [testcore]
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:828)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:751)
   [junit4]    > 	... 41 more
   [junit4]    > Caused by: org.apache.solr.common.SolrException: Timed out getting shard id for core: testcore
   [junit4]    > 	at org.apache.solr.cloud.ZkController.waitForShardId(ZkController.java:1407)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1362)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
   [junit4]    > 	... 42 more
   [junit4]   2> 5290386 INFO  (SUITE-ClusterStateUpdateTest-seed#[7FC47D48B54585CC]-worker) [n:127.0.0.1:1661_solr    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ClusterStateUpdateTest_7FC47D48B54585CC-001
   [junit4]   2> Jun 06, 2016 5:51:26 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 3 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene60, sim=ClassicSimilarity, locale=hi, timezone=Asia/Hebron
   [junit4]   2> NOTE: Linux 4.4.0-21-generic amd64/Oracle Corporation 1.8.0_92 (64-bit)/cpus=12,threads=1,free=182924104,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [CollectionsAPIDistributedZkTest, TestPivotHelperCode, BlockDirectoryTest, UniqFieldsUpdateProcessorFactoryTest, TestScoreJoinQPNoScore, FacetPivotSmallTest, FastVectorHighlighterTest, TestSolrDeletionPolicy2, HdfsThreadLeakTest, CSVRequestHandlerTest, TestSerializedLuceneMatchVersion, DocumentBuilderTest, SecurityConfHandlerTest, SuggestComponentTest, TestDynamicFieldResource, HdfsCollectionsAPIDistributedZkTest, TestEmbeddedSolrServerConstructors, TestChildDocTransformer, TestSortByMinMaxFunction, NumericFieldsTest, TestCodecSupport, SoftAutoCommitTest, HttpPartitionTest, TestCoreContainer, TestRestManager, TestFastLRUCache, DistribCursorPagingTest, TestQueryUtils, TestStressUserVersions, HighlighterMaxOffsetTest, TestQuerySenderListener, NotRequiredUniqueKeyTest, TestLegacyFieldCache, ShardRoutingCustomTest, MoreLikeThisHandlerTest, LoggingHandlerTest, CursorMarkTest, CloudMLTQParserTest, TestLMJelinekMercerSimilarityFactory, OpenExchangeRatesOrgProviderTest, DistributedQueueTest, TestAuthorizationFramework, TestCloudDeleteByQuery, SpellCheckComponentTest, DocValuesMissingTest, MinimalSchemaTest, SolrXmlInZkTest, ShardRoutingTest, BasicFunctionalityTest, CloneFieldUpdateProcessorFactoryTest, ChaosMonkeyNothingIsSafeTest, TestCloudSchemaless, TestFreeTextSuggestions, TestInfoStreamLogging, TestDocumentBuilder, TestQueryWrapperFilter, SortByFunctionTest, TestDFRSimilarityFactory, CollectionReloadTest, TestFiltering, TestSolrCloudWithKerberosAlt, TestMissingGroups, TestSubQueryTransformerDistrib, DistanceFunctionTest, SolrCmdDistributorTest, CdcrUpdateLogTest, RecoveryAfterSoftCommitTest, TermsComponentTest, TestUpdate, HdfsWriteToMultipleCollectionsTest, TestGeoJSONResponseWriter, DistributedExpandComponentTest, ConfigSetsAPITest, TestStressVersions, SolrInfoMBeanTest, DistributedQueryComponentCustomSortTest, HighlighterTest, DistributedTermsComponentTest, ZkSolrClientTest, TestSSLRandomization, TestDistributedGrouping, BitVectorTest, CoreAdminRequestStatusTest, TestTestInjection, TestBulkSchemaConcurrent, MultiTermTest, TestLRUStatsCache, TestJsonFacets, TestBM25SimilarityFactory, TestRawTransformer, TestIntervalFaceting, TimeZoneUtilsTest, AddSchemaFieldsUpdateProcessorFactoryTest, HdfsRecoveryZkTest, TestManagedSchemaAPI, TestTolerantUpdateProcessorCloud, TestSearcherReuse, ClusterStateUpdateTest]
   [junit4] Completed [611/613 (9!)] on J1 in 322.28s, 1 test, 1 error <<< FAILURES!

[...truncated 26 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 2 suite-level errors, 5 errors, 5 failures, 1517 ignored (58 assumptions) [seed: 7FC47D48B54585CC]

Total time: 109 minutes 5 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS-EA] Lucene-Solr-6.x-Linux (32bit/jdk-9-ea+121) - Build # 835 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/835/
Java: 32bit/jdk-9-ea+121 -client -XX:+UseG1GC

20 tests failed.
FAILED:  org.apache.solr.core.TestDynamicLoading.testDynamicLoading

Error Message:
Could not get expected value  'X val' for path 'x' full output: {   "responseHeader":{     "status":0,     "QTime":0},   "params":{"wt":"json"},   "context":{     "webapp":"/n_dxi",     "path":"/test1",     "httpMethod":"GET"},   "class":"org.apache.solr.core.BlobStoreTestRequestHandler",   "x":null},  from server:  null

Stack Trace:
java.lang.AssertionError: Could not get expected value  'X val' for path 'x' full output: {
  "responseHeader":{
    "status":0,
    "QTime":0},
  "params":{"wt":"json"},
  "context":{
    "webapp":"/n_dxi",
    "path":"/test1",
    "httpMethod":"GET"},
  "class":"org.apache.solr.core.BlobStoreTestRequestHandler",
  "x":null},  from server:  null
	at __randomizedtesting.SeedInfo.seed([DAC5F8AEBEB053BE:288D5F9496DF61E]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.core.TestSolrConfigHandler.testForResponseElement(TestSolrConfigHandler.java:457)
	at org.apache.solr.core.TestDynamicLoading.testDynamicLoading(TestDynamicLoading.java:233)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.schema.TestBulkSchemaConcurrent.test

Error Message:
[[], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], [   {     "replace-field":{       "name":"a1",       "type":"text",       "stored":true,       "indexed":true},     "errorMessages":["The field 'a1' is not present in this schema, and so cannot be replaced.\n"]},   {     "replace-dynamic-field":{       "name":"*_lol1",       "type":"text",       "stored":true,       "indexed":true},     "errorMessages":["The dynamic field '*_lol1' is not present in this schema, and so cannot be replaced.\n"]},   {     "replace-field-type":{       "name":"mystr1",       "class":"solr.TextField"},     "errorMessages":["The field type 'mystr1' is not present in this schema, and so cannot be replaced.\n"]}]], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], [   {     "replace-field":{       "name":"a2",       "type":"text",       "stored":true,       "indexed":true},     "errorMessages":["The field 'a2' is not present in this schema, and so cannot be replaced.\n"]},   {     "replace-dynamic-field":{       "name":"*_lol2",       "type":"text",       "stored":true,       "indexed":true},     "errorMessages":["The dynamic field '*_lol2' is not present in this schema, and so cannot be replaced.\n"]},   {     "replace-field-type":{       "name":"mystr2",       "class":"solr.TextField"},     "errorMessages":["The field type 'mystr2' is not present in this schema, and so cannot be replaced.\n"]}]], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], [   {     "replace-field":{       "name":"a3",       "type":"text",       "stored":true,       "indexed":true},     "errorMessages":["The field 'a3' is not present in this schema, and so cannot be replaced.\n"]},   {     "replace-dynamic-field":{       "name":"*_lol3",       "type":"text",       "stored":true,       "indexed":true},     "errorMessages":["The dynamic field '*_lol3' is not present in this schema, and so cannot be replaced.\n"]},   {     "replace-field-type":{       "name":"mystr3",       "class":"solr.TextField"},     "errorMessages":["The field type 'mystr3' is not present in this schema, and so cannot be replaced.\n"]}]], []]

Stack Trace:
java.lang.AssertionError: [[], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], [
  {
    "replace-field":{
      "name":"a1",
      "type":"text",
      "stored":true,
      "indexed":true},
    "errorMessages":["The field 'a1' is not present in this schema, and so cannot be replaced.\n"]},
  {
    "replace-dynamic-field":{
      "name":"*_lol1",
      "type":"text",
      "stored":true,
      "indexed":true},
    "errorMessages":["The dynamic field '*_lol1' is not present in this schema, and so cannot be replaced.\n"]},
  {
    "replace-field-type":{
      "name":"mystr1",
      "class":"solr.TextField"},
    "errorMessages":["The field type 'mystr1' is not present in this schema, and so cannot be replaced.\n"]}]], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], [
  {
    "replace-field":{
      "name":"a2",
      "type":"text",
      "stored":true,
      "indexed":true},
    "errorMessages":["The field 'a2' is not present in this schema, and so cannot be replaced.\n"]},
  {
    "replace-dynamic-field":{
      "name":"*_lol2",
      "type":"text",
      "stored":true,
      "indexed":true},
    "errorMessages":["The dynamic field '*_lol2' is not present in this schema, and so cannot be replaced.\n"]},
  {
    "replace-field-type":{
      "name":"mystr2",
      "class":"solr.TextField"},
    "errorMessages":["The field type 'mystr2' is not present in this schema, and so cannot be replaced.\n"]}]], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], [
  {
    "replace-field":{
      "name":"a3",
      "type":"text",
      "stored":true,
      "indexed":true},
    "errorMessages":["The field 'a3' is not present in this schema, and so cannot be replaced.\n"]},
  {
    "replace-dynamic-field":{
      "name":"*_lol3",
      "type":"text",
      "stored":true,
      "indexed":true},
    "errorMessages":["The dynamic field '*_lol3' is not present in this schema, and so cannot be replaced.\n"]},
  {
    "replace-field-type":{
      "name":"mystr3",
      "class":"solr.TextField"},
    "errorMessages":["The field type 'mystr3' is not present in this schema, and so cannot be replaced.\n"]}]], []]
	at __randomizedtesting.SeedInfo.seed([DAC5F8AEBEB053BE:5291C774104C3E46]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.schema.TestBulkSchemaConcurrent.test(TestBulkSchemaConcurrent.java:115)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest:     1) Thread[id=8675, name=Thread-3264, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    2) Thread[id=8643, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[DAC5F8AEBEB053BE]-SendThread(127.0.0.1:34642), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    3) Thread[id=8669, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[DAC5F8AEBEB053BE]-SendThread(127.0.0.1:39497), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    4) Thread[id=8678, name=zkCallback-1602-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    5) Thread[id=8648, name=OverseerHdfsCoreFailoverThread-96021316510941187-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    6) Thread[id=8644, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[DAC5F8AEBEB053BE]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    7) Thread[id=8649, name=Thread-3262, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    8) Thread[id=8673, name=OverseerCollectionConfigSetProcessor-96021337503170563-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=8646, name=OverseerStateUpdate-96021316510941187-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   10) Thread[id=8677, name=zkCallback-1602-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   11) Thread[id=8680, name=zkCallback-1602-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   12) Thread[id=8679, name=zkCallback-1602-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   13) Thread[id=8672, name=OverseerStateUpdate-96021337503170563-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)         at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   14) Thread[id=8670, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[DAC5F8AEBEB053BE]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   15) Thread[id=8674, name=OverseerHdfsCoreFailoverThread-96021337503170563-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   16) Thread[id=8647, name=OverseerCollectionConfigSetProcessor-96021316510941187-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest: 
   1) Thread[id=8675, name=Thread-3264, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   2) Thread[id=8643, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[DAC5F8AEBEB053BE]-SendThread(127.0.0.1:34642), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   3) Thread[id=8669, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[DAC5F8AEBEB053BE]-SendThread(127.0.0.1:39497), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   4) Thread[id=8678, name=zkCallback-1602-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   5) Thread[id=8648, name=OverseerHdfsCoreFailoverThread-96021316510941187-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   6) Thread[id=8644, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[DAC5F8AEBEB053BE]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   7) Thread[id=8649, name=Thread-3262, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   8) Thread[id=8673, name=OverseerCollectionConfigSetProcessor-96021337503170563-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=8646, name=OverseerStateUpdate-96021316510941187-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  10) Thread[id=8677, name=zkCallback-1602-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  11) Thread[id=8680, name=zkCallback-1602-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  12) Thread[id=8679, name=zkCallback-1602-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  13) Thread[id=8672, name=OverseerStateUpdate-96021337503170563-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
        at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)
        at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)
        at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  14) Thread[id=8670, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[DAC5F8AEBEB053BE]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  15) Thread[id=8674, name=OverseerHdfsCoreFailoverThread-96021337503170563-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  16) Thread[id=8647, name=OverseerCollectionConfigSetProcessor-96021316510941187-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
	at __randomizedtesting.SeedInfo.seed([DAC5F8AEBEB053BE]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=8675, name=Thread-3264, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    2) Thread[id=8677, name=zkCallback-1602-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    3) Thread[id=8643, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[DAC5F8AEBEB053BE]-SendThread(127.0.0.1:34642), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    4) Thread[id=8669, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[DAC5F8AEBEB053BE]-SendThread(127.0.0.1:39497), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    5) Thread[id=8678, name=zkCallback-1602-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    6) Thread[id=8680, name=zkCallback-1602-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=8679, name=zkCallback-1602-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    8) Thread[id=8648, name=OverseerHdfsCoreFailoverThread-96021316510941187-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=8674, name=OverseerHdfsCoreFailoverThread-96021337503170563-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   10) Thread[id=8649, name=Thread-3262, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=8675, name=Thread-3264, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   2) Thread[id=8677, name=zkCallback-1602-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   3) Thread[id=8643, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[DAC5F8AEBEB053BE]-SendThread(127.0.0.1:34642), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   4) Thread[id=8669, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[DAC5F8AEBEB053BE]-SendThread(127.0.0.1:39497), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   5) Thread[id=8678, name=zkCallback-1602-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   6) Thread[id=8680, name=zkCallback-1602-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   7) Thread[id=8679, name=zkCallback-1602-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   8) Thread[id=8648, name=OverseerHdfsCoreFailoverThread-96021316510941187-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=8674, name=OverseerHdfsCoreFailoverThread-96021337503170563-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  10) Thread[id=8649, name=Thread-3262, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
	at __randomizedtesting.SeedInfo.seed([DAC5F8AEBEB053BE]:0)


FAILED:  org.apache.solr.cloud.AsyncMigrateRouteKeyTest.test

Error Message:
Error from server at http://127.0.0.1:36719/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:36719/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([DAC5F8AEBEB053BE:5291C774104C3E46]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest(MigrateRouteKeyTest.java:135)
	at org.apache.solr.cloud.AsyncMigrateRouteKeyTest.test(AsyncMigrateRouteKeyTest.java:44)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
Error from server at http://127.0.0.1:42284: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore

Stack Trace:
java.lang.AssertionError: Error from server at http://127.0.0.1:42284: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore
	at __randomizedtesting.SeedInfo.seed([DAC5F8AEBEB053BE:5291C774104C3E46]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:162)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:71)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:38795/ny/y/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /ny/y/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:38795/ny/y/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /ny/y/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([DAC5F8AEBEB053BE:5291C774104C3E46]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:152)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
Error from server at https://127.0.0.1:39709/_/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /_/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:39709/_/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /_/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create

[...truncated too long message...]

t failure: Timed out getting coreNodeName for collection1
   [junit4]    > 	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
   [junit4]    > 	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
   [junit4]    > 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
   [junit4]    > 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]    > 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
   [junit4]    > 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]    > 	at org.eclipse.jetty.server.Server.handle(Server.java:518)
   [junit4]    > 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
   [junit4]    > 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
   [junit4]    > 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]    > 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]    > 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]    > 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
   [junit4]    > 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    > Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
   [junit4]    > 	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
   [junit4]    > 	... 1 more
   [junit4]    > ,code=500}</pre></p>
   [junit4]    > <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
   [junit4]    > </body>
   [junit4]    > </html>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([DAC5F8AEBEB053BE:5291C774104C3E46]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.indexDoc(AbstractFullDistribZkTestBase.java:775)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.index(BaseDistributedSearchTestCase.java:483)
   [junit4]    > 	at org.apache.solr.cloud.TestShortCircuitedRequests.test(TestShortCircuitedRequests.java:39)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]   2> 9608726 INFO  (SUITE-TestShortCircuitedRequests-seed#[DAC5F8AEBEB053BE]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestShortCircuitedRequests_DAC5F8AEBEB053BE-001
   [junit4]   2> Jun 05, 2016 11:58:43 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 5 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene60): {}, docValues:{}, maxPointsInLeafNode=1384, maxMBSortInHeap=7.003051398770211, sim=ClassicSimilarity, locale=ar-SD, timezone=America/Chihuahua
   [junit4]   2> NOTE: Linux 4.4.0-21-generic i386/Oracle Corporation 9-ea (32-bit)/cpus=12,threads=1,free=38539992,total=232783872
   [junit4]   2> NOTE: All tests run in this JVM: [AddBlockUpdateTest, ShowFileRequestHandlerTest, TestRequestForwarding, HardAutoCommitTest, TestDynamicLoading, TestSolrFieldCacheMBean, TestSchemaManager, TestCollationField, ChangedSchemaMergeTest, TestFieldCollectionResource, HdfsChaosMonkeyNothingIsSafeTest, TestScoreJoinQPScore, TestXIncludeConfig, TestStressRecovery, TestTrieFacet, TestHighFrequencyDictionaryFactory, AssignTest, TestComplexPhraseQParserPlugin, ExplicitHLLTest, TestExclusionRuleCollectionAccess, TestSuggestSpellingConverter, UUIDUpdateProcessorFallbackTest, TestDocTermOrds, TestSolrConfigHandlerConcurrent, TestTolerantSearch, TestCharFilters, CircularListTest, TestSlowCompositeReaderWrapper, RequestHandlersTest, TestManagedResource, HighlighterConfigTest, TestFieldCacheVsDocValues, DeleteInactiveReplicaTest, TestNumericTerms32, SuggesterTSTTest, TestExactSharedStatsCache, TestHdfsUpdateLog, TestFilteredDocIdSet, DebugComponentTest, XmlUpdateRequestHandlerTest, TestAddFieldRealTimeGet, TestRecoveryHdfs, TestPHPSerializedResponseWriter, TestWriterPerf, TestConfigReload, DirectUpdateHandlerOptimizeTest, SuggesterWFSTTest, TestCloudInspectUtil, SolrTestCaseJ4Test, RemoteQueryErrorTest, CollectionStateFormat2Test, QueryParsingTest, CoreAdminHandlerTest, TestJmxMonitoredMap, TestSolrQueryParser, TestStressCloudBlindAtomicUpdates, SimpleCollectionCreateDeleteTest, TestNonDefinedSimilarityFactory, RulesTest, BJQParserTest, TestNoOpRegenerator, DistributedVersionInfoTest, TestFieldTypeResource, HdfsTlogReplayBufferedWhileIndexingTest, RollingRestartTest, TestExtendedDismaxParser, TestReversedWildcardFilterFactory, AutoCommitTest, TestIndexingPerformance, SampleTest, SuggesterTest, BasicDistributedZk2Test, StressHdfsTest, TestSchemaSimilarityResource, TestInitParams, DistributedQueryComponentOptimizationTest, LukeRequestHandlerTest, TestManagedStopFilterFactory, UnloadDistributedZkTest, TestShortCircuitedRequests]
   [junit4] Completed [594/613 (17!)] on J2 in 604.12s, 1 test, 1 error <<< FAILURES!

[...truncated 108 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 2 suite-level errors, 13 errors, 5 failures, 1258 ignored (69 assumptions) [seed: DAC5F8AEBEB053BE]

Total time: 206 minutes 31 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS-EA] Lucene-Solr-6.x-Linux (64bit/jdk-9-ea+121) - Build # 834 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/834/
Java: 64bit/jdk-9-ea+121 -XX:+UseCompressedOops -XX:+UseParallelGC

25 tests failed.
FAILED:  org.apache.solr.schema.TestCloudSchemaless.test

Error Message:
QUERY FAILED: xpath=/response/arr[@name='fields']/lst/str[@name='name'][.='newTestFieldInt6']  request=/schema/fields?wt=xml  response=<?xml version="1.0" encoding="UTF-8"?> <response> <lst name="responseHeader"><int name="status">0</int><int name="QTime">1</int></lst><arr name="fields"><lst><str name="name">_root_</str><str name="type">string</str><bool name="multiValued">false</bool><bool name="indexed">true</bool><bool name="stored">true</bool></lst><lst><str name="name">_version_</str><str name="type">long</str><bool name="indexed">true</bool><bool name="stored">true</bool></lst><lst><str name="name">constantField</str><str name="type">tdouble</str></lst><lst><str name="name">id</str><str name="type">string</str><bool name="multiValued">false</bool><bool name="indexed">true</bool><bool name="required">true</bool><bool name="stored">true</bool></lst><lst><str name="name">newTestFieldInt0</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt1</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt10</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt100</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt101</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt102</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt103</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt104</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt105</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt106</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt107</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt108</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt109</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt11</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt110</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt111</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt112</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt113</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt114</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt115</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt116</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt117</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt118</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt119</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt12</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt120</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt121</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt122</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt123</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt124</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt125</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt126</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt127</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt128</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt129</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt13</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt130</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt131</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt132</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt133</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt134</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt135</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt136</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt137</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt138</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt139</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt14</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt140</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt141</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt142</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt143</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt144</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt145</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt146</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt147</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt148</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt149</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt15</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt150</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt151</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt152</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt153</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt154</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt155</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt156</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt157</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt158</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt159</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt16</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt160</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt161</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt162</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt163</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt164</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt165</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt166</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt167</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt168</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt169</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt17</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt170</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt171</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt172</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt173</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt174</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt175</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt176</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt177</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt178</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt179</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt18</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt180</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt181</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt182</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt183</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt184</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt185</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt186</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt187</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt188</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt189</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt19</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt190</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt191</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt192</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt193</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt194</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt195</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt196</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt197</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt198</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt199</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt2</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt20</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt200</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt201</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt202</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt203</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt204</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt205</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt206</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt207</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt208</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt209</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt21</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt210</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt211</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt212</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt213</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt214</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt215</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt216</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt217</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt218</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt219</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt22</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt220</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt221</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt222</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt223</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt224</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt225</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt226</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt227</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt228</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt229</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt23</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt230</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt231</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt232</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt233</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt234</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt235</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt236</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt237</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt238</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt239</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt24</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt240</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt241</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt242</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt243</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt244</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt245</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt246</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt247</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt248</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt249</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt25</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt250</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt251</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt252</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt253</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt254</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt255</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt256</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt257</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt258</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt259</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt26</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt260</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt261</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt262</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt263</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt264</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt265</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt266</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt267</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt268</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt269</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt27</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt270</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt271</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt272</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt273</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt274</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt275</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt276</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt277</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt278</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt279</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt28</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt280</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt281</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt282</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt283</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt284</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt285</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt286</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt287</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt288</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt289</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt29</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt290</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt291</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt292</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt293</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt294</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt295</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt296</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt297</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt298</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt299</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt3</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt30</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt300</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt301</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt302</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt303</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt304</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt305</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt306</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt307</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt308</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt309</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt31</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt310</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt311</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt312</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt313</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt314</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt315</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt316</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt317</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt318</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt319</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt32</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt320</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt321</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt322</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt323</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt324</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt325</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt326</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt327</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt328</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt329</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt33</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt330</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt331</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt332</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt333</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt334</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt335</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt336</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt337</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt338</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt339</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt34</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt340</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt341</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt342</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt343</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt344</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt345</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt346</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt347</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt348</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt349</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt35</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt350</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt351</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt352</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt353</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt354</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt355</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt356</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt357</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt358</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt359</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt36</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt360</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt361</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt362</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt363</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt364</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt365</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt366</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt367</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt368</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt369</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt37</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt370</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt371</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt372</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt373</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt374</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt375</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt376</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt377</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt378</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt379</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt38</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt380</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt381</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt382</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt383</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt384</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt385</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt386</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt387</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt388</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt389</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt39</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt390</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt391</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt392</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt393</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt394</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt395</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt396</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt397</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt398</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt399</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt4</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt40</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt400</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt401</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt402</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt403</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt404</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt405</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt406</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt407</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt408</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt409</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt41</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt410</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt411</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt412</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt413</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt414</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt415</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt416</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt417</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt418</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt419</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt42</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt420</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt421</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt422</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt423</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt424</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt425</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt426</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt427</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt428</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt429</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt43</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt430</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt431</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt432</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt433</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt434</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt435</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt436</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt437</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt438</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt439</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt44</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt440</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt441</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt442</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt443</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt444</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt445</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt446</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt447</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt448</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt449</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt45</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt46</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt47</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt48</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt49</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt5</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt50</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt51</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt52</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt53</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt54</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt55</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt56</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt57</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt58</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt59</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt60</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt61</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt62</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt63</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt64</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt65</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt66</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt67</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt68</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt69</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt7</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt70</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt71</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt72</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt73</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt74</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt75</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt76</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt77</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt78</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt79</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt8</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt80</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt81</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt82</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt83</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt84</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt85</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt86</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt87</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt88</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt89</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt9</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt90</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt91</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt92</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt93</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt94</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt95</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt96</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt97</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt98</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt99</str><str name="type">tlong</str></lst></arr> </response> 

Stack Trace:
java.lang.AssertionError: QUERY FAILED: xpath=/response/arr[@name='fields']/lst/str[@name='name'][.='newTestFieldInt6']  request=/schema/fields?wt=xml  response=<?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">1</int></lst><arr name="fields"><lst><str name="name">_root_</str><str name="type">string</str><bool name="multiValued">false</bool><bool name="indexed">true</bool><bool name="stored">true</bool></lst><lst><str name="name">_version_</str><str name="type">long</str><bool name="indexed">true</bool><bool name="stored">true</bool></lst><lst><str name="name">constantField</str><str name="type">tdouble</str></lst><lst><str name="name">id</str><str name="type">string</str><bool name="multiValued">false</bool><bool name="indexed">true</bool><bool name="required">true</bool><bool name="stored">true</bool></lst><lst><str name="name">newTestFieldInt0</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt1</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt10</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt100</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt101</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt102</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt103</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt104</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt105</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt106</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt107</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt108</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt109</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt11</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt110</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt111</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt112</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt113</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt114</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt115</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt116</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt117</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt118</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt119</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt12</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt120</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt121</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt122</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt123</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt124</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt125</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt126</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt127</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt128</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt129</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt13</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt130</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt131</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt132</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt133</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt134</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt135</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt136</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt137</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt138</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt139</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt14</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt140</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt141</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt142</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt143</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt144</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt145</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt146</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt147</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt148</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt149</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt15</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt150</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt151</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt152</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt153</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt154</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt155</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt156</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt157</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt158</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt159</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt16</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt160</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt161</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt162</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt163</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt164</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt165</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt166</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt167</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt168</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt169</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt17</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt170</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt171</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt172</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt173</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt174</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt175</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt176</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt177</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt178</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt179</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt18</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt180</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt181</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt182</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt183</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt184</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt185</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt186</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt187</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt188</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt189</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt19</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt190</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt191</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt192</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt193</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt194</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt195</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt196</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt197</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt198</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt199</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt2</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt20</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt200</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt201</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt202</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt203</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt204</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt205</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt206</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt207</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt208</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt209</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt21</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt210</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt211</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt212</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt213</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt214</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt215</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt216</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt217</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt218</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt219</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt22</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt220</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt221</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt222</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt223</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt224</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt225</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt226</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt227</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt228</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt229</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt23</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt230</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt231</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt232</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt233</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt234</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt235</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt236</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt237</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt238</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt239</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt24</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt240</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt241</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt242</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt243</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt244</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt245</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt246</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt247</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt248</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt249</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt25</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt250</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt251</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt252</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt253</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt254</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt255</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt256</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt257</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt258</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt259</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt26</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt260</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt261</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt262</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt263</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt264</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt265</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt266</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt267</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt268</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt269</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt27</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt270</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt271</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt272</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt273</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt274</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt275</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt276</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt277</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt278</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt279</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt28</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt280</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt281</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt282</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt283</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt284</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt285</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt286</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt287</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt288</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt289</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt29</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt290</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt291</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt292</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt293</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt294</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt295</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt296</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt297</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt298</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt299</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt3</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt30</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt300</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt301</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt302</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt303</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt304</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt305</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt306</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt307</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt308</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt309</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt31</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt310</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt311</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt312</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt313</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt314</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt315</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt316</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt317</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt318</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt319</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt32</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt320</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt321</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt322</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt323</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt324</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt325</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt326</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt327</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt328</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt329</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt33</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt330</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt331</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt332</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt333</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt334</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt335</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt336</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt337</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt338</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt339</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt34</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt340</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt341</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt342</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt343</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt344</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt345</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt346</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt347</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt348</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt349</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt35</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt350</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt351</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt352</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt353</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt354</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt355</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt356</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt357</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt358</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt359</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt36</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt360</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt361</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt362</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt363</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt364</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt365</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt366</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt367</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt368</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt369</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt37</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt370</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt371</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt372</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt373</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt374</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt375</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt376</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt377</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt378</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt379</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt38</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt380</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt381</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt382</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt383</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt384</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt385</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt386</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt387</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt388</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt389</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt39</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt390</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt391</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt392</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt393</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt394</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt395</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt396</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt397</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt398</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt399</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt4</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt40</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt400</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt401</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt402</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt403</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt404</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt405</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt406</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt407</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt408</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt409</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt41</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt410</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt411</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt412</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt413</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt414</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt415</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt416</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt417</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt418</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt419</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt42</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt420</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt421</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt422</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt423</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt424</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt425</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt426</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt427</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt428</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt429</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt43</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt430</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt431</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt432</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt433</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt434</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt435</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt436</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt437</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt438</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt439</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt44</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt440</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt441</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt442</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt443</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt444</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt445</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt446</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt447</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt448</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt449</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt45</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt46</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt47</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt48</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt49</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt5</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt50</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt51</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt52</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt53</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt54</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt55</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt56</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt57</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt58</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt59</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt60</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt61</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt62</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt63</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt64</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt65</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt66</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt67</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt68</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt69</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt7</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt70</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt71</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt72</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt73</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt74</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt75</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt76</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt77</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt78</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt79</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt8</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt80</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt81</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt82</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt83</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt84</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt85</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt86</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt87</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt88</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt89</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt9</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt90</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt91</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt92</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt93</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt94</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt95</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt96</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt97</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt98</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt99</str><str name="type">tlong</str></lst></arr>
</response>

	at __randomizedtesting.SeedInfo.seed([56E77ED5C0190ED7:DEB3410F6EE5632F]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.schema.TestCloudSchemaless.test(TestCloudSchemaless.java:139)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest:     1) Thread[id=6587, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[56E77ED5C0190ED7]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    2) Thread[id=6614, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[56E77ED5C0190ED7]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    3) Thread[id=6619, name=Thread-1882, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    4) Thread[id=6623, name=zkCallback-1401-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    5) Thread[id=6613, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[56E77ED5C0190ED7]-SendThread(127.0.0.1:41242), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    6) Thread[id=6622, name=zkCallback-1401-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=6617, name=OverseerCollectionConfigSetProcessor-96020038195347459-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    8) Thread[id=6592, name=Thread-1880, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    9) Thread[id=6616, name=OverseerStateUpdate-96020038195347459-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)         at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   10) Thread[id=6624, name=zkCallback-1401-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   11) Thread[id=6618, name=OverseerHdfsCoreFailoverThread-96020038195347459-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   12) Thread[id=6621, name=zkCallback-1401-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   13) Thread[id=6591, name=OverseerHdfsCoreFailoverThread-96020017207312387-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   14) Thread[id=6589, name=OverseerStateUpdate-96020017207312387-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   15) Thread[id=6586, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[56E77ED5C0190ED7]-SendThread(127.0.0.1:38500), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)   16) Thread[id=6590, name=OverseerCollectionConfigSetProcessor-96020017207312387-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest: 
   1) Thread[id=6587, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[56E77ED5C0190ED7]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   2) Thre

[...truncated too long message...]

n/96020058430242830-core_node2-n_0000000002
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:244)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:241)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:241)
   [junit4]   2> 	at org.apache.solr.cloud.ElectionContext.cancelElection(ElectionContext.java:86)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContextBase.cancelElection(ElectionContext.java:143)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.cancelElection(ElectionContext.java:279)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:455)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(java.base@9-ea/Executors.java:514)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
   [junit4]   2> 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]   2> 10577119 INFO  (TEST-RecoveryZkTest.test-seed#[56E77ED5C0190ED7]) [    ] o.e.j.s.ServerConnector Stopped ServerConnector@6e562c3b{HTTP/1.1,[http/1.1]}{127.0.0.1:43507}
   [junit4]   2> 10577119 INFO  (TEST-RecoveryZkTest.test-seed#[56E77ED5C0190ED7]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1684a0c{/qz_q/x,null,UNAVAILABLE}
   [junit4]   2> 10577119 INFO  (TEST-RecoveryZkTest.test-seed#[56E77ED5C0190ED7]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45966 45966
   [junit4]   2> 10577144 INFO  (Thread-3279) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45966 45966
   [junit4]   2> 10577144 WARN  (Thread-3279) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/aliases.json
   [junit4]   2> 	5	/solr/clusterprops.json
   [junit4]   2> 	4	/solr/security.json
   [junit4]   2> 	3	/solr/collections/collection1/state.json
   [junit4]   2> 	3	/solr/configs/conf1
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/overseer_elect/election/96020058430242824-127.0.0.1:34056_qz_q%2Fx-n_0000000001
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/collections
   [junit4]   2> 	4	/solr/live_nodes
   [junit4]   2> 	3	/solr/overseer/queue
   [junit4]   2> 	3	/solr/overseer/collection-queue-work
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryZkTest -Dtests.method=test -Dtests.seed=56E77ED5C0190ED7 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=en-SX -Dtests.timezone=Africa/Abidjan -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE  626s J0 | RecoveryZkTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([56E77ED5C0190ED7:DEB3410F6EE5632F]:0)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:127)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]   2> 10577145 INFO  (SUITE-RecoveryZkTest-seed#[56E77ED5C0190ED7]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest_56E77ED5C0190ED7-001
   [junit4]   2> Jun 05, 2016 6:14:03 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene60): {}, docValues:{}, maxPointsInLeafNode=908, maxMBSortInHeap=7.330895656806706, sim=RandomSimilarity(queryNorm=false,coord=yes): {}, locale=en-SX, timezone=Africa/Abidjan
   [junit4]   2> NOTE: Linux 4.4.0-21-generic amd64/Oracle Corporation 9-ea (64-bit)/cpus=12,threads=1,free=96044600,total=455081984
   [junit4]   2> NOTE: All tests run in this JVM: [TestDistribDocBasedVersion, DeleteStatusTest, ConnectionReuseTest, IndexSchemaRuntimeFieldTest, InfoHandlerTest, TestCustomDocTransformer, TestImplicitCoreProperties, TestLMDirichletSimilarityFactory, TestSolrQueryParserResource, OverseerTaskQueueTest, PingRequestHandlerTest, TestGraphTermsQParserPlugin, AnalyticsQueryTest, PolyFieldTest, PKIAuthenticationIntegrationTest, TestConfig, ForceLeaderTest, UUIDFieldTest, TestDocBasedVersionConstraints, TestRawResponseWriter, TestRangeQuery, TestLazyCores, TestFoldingMultitermQuery, MultiThreadedOCPTest, PathHierarchyTokenizerFactoryTest, URLClassifyProcessorTest, TestSchemaVersionResource, SystemInfoHandlerTest, TestMiniSolrCloudClusterSSL, HdfsRecoverLeaseTest, TestCloudPivotFacet, BasicZkTest, IndexSchemaTest, RAMDirectoryFactoryTest, SynonymTokenizerTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, VersionInfoTest, SparseHLLTest, TestSQLHandler, DistributedSpellCheckComponentTest, QueryEqualityTest, DistanceUnitsTest, TestManagedSchema, TestDistributedSearch, TestCustomSort, DistributedSuggestComponentTest, TestCrossCoreJoin, OverseerCollectionConfigSetProcessorTest, CdcrVersionReplicationTest, TestValueSourceCache, TestComponentsName, TestCloudBackupRestore, BigEndianAscendingWordSerializerTest, EchoParamsTest, LeaderInitiatedRecoveryOnCommitTest, TestConfigSetsAPIExclusivity, TestLRUCache, SuggestComponentContextFilterQueryTest, FileBasedSpellCheckerTest, CreateCollectionCleanupTest, TestUseDocValuesAsStored, TestStressLucene, TestLuceneMatchVersion, TestOmitPositions, TestOverriddenPrefixQueryForCustomFieldType, TestLeaderElectionZkExpiry, SignatureUpdateProcessorFactoryTest, TestDocSet, TestCSVLoader, HdfsChaosMonkeySafeLeaderTest, ResourceLoaderTest, TestSolrConfigHandler, TestFaceting, TestDefaultStatsCache, TestMiniSolrCloudCluster, TestFastOutputStream, ConjunctionSolrSpellCheckerTest, TestExactStatsCache, TestClusterStateMutator, DistributedFacetPivotLongTailTest, TlogReplayBufferedWhileIndexingTest, TestFieldResource, TestManagedSynonymFilterFactory, TestSolr4Spatial, DocumentAnalysisRequestHandlerTest, HdfsSyncSliceTest, TestNamedUpdateProcessors, SSLMigrationTest, StandardRequestHandlerTest, TestAnalyzedSuggestions, TestRandomDVFaceting, QueryResultKeyTest, PrimUtilsTest, TestRemoteStreaming, DeleteLastCustomShardedReplicaTest, GraphQueryTest, AsyncCallRequestStatusResponseTest, TestClassicSimilarityFactory, BasicDistributedZkTest, UnloadDistributedZkTest, SyncSliceTest, ShardRoutingTest, RecoveryZkTest]
   [junit4] Completed [613/613 (22!)] on J0 in 626.76s, 1 test, 1 failure <<< FAILURES!

[...truncated 20 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 2 suite-level errors, 18 errors, 5 failures, 1199 ignored (71 assumptions) [seed: 56E77ED5C0190ED7]

Total time: 196 minutes 36 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS-EA] Lucene-Solr-6.x-Linux (64bit/jdk-9-ea+121) - Build # 833 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/833/
Java: 64bit/jdk-9-ea+121 -XX:-UseCompressedOops -XX:+UseG1GC

21 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
17 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest:     1) Thread[id=5978, name=zkCallback-1107-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    2) Thread[id=5945, name=Thread-2171, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    3) Thread[id=5974, name=zkCallback-1107-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    4) Thread[id=5972, name=Thread-2173, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    5) Thread[id=5944, name=OverseerHdfsCoreFailoverThread-96018484180746243-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    6) Thread[id=5976, name=zkCallback-1107-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=5971, name=OverseerHdfsCoreFailoverThread-96018505175793667-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    8) Thread[id=5967, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[49FB930F5CA93324]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    9) Thread[id=5970, name=OverseerCollectionConfigSetProcessor-96018505175793667-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   10) Thread[id=5940, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[49FB930F5CA93324]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   11) Thread[id=5943, name=OverseerCollectionConfigSetProcessor-96018484180746243-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   12) Thread[id=5939, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[49FB930F5CA93324]-SendThread(127.0.0.1:44025), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)   13) Thread[id=5977, name=zkCallback-1107-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   14) Thread[id=5969, name=OverseerStateUpdate-96018505175793667-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)         at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   15) Thread[id=5966, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[49FB930F5CA93324]-SendThread(127.0.0.1:35001), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   16) Thread[id=5975, name=zkCallback-1107-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   17) Thread[id=5942, name=OverseerStateUpdate-96018484180746243-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 17 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest: 
   1) Thread[id=5978, name=zkCallback-1107-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   2) Thread[id=5945, name=Thread-2171, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   3) Thread[id=5974, name=zkCallback-1107-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   4) Thread[id=5972, name=Thread-2173, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   5) Thread[id=5944, name=OverseerHdfsCoreFailoverThread-96018484180746243-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   6) Thread[id=5976, name=zkCallback-1107-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   7) Thread[id=5971, name=OverseerHdfsCoreFailoverThread-96018505175793667-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   8) Thread[id=5967, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[49FB930F5CA93324]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   9) Thread[id=5970, name=OverseerCollectionConfigSetProcessor-96018505175793667-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  10) Thread[id=5940, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[49FB930F5CA93324]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  11) Thread[id=5943, name=OverseerCollectionConfigSetProcessor-96018484180746243-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  12) Thread[id=5939, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[49FB930F5CA93324]-SendThread(127.0.0.1:44025), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
  13) Thread[id=5977, name=zkCallback-1107-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  14) Thread[id=5969, name=OverseerStateUpdate-96018505175793667-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
        at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)
        at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)
        at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  15) Thread[id=5966, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[49FB930F5CA93324]-SendThread(127.0.0.1:35001), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  16) Thread[id=5975, name=zkCallback-1107-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  17) Thread[id=5942, name=OverseerStateUpdate-96018484180746243-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=5978, name=zkCallback-1107-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    2) Thread[id=5939, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[49FB930F5CA93324]-SendThread(127.0.0.1:44025), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    3) Thread[id=5945, name=Thread-2171, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    4) Thread[id=5974, name=zkCallback-1107-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    5) Thread[id=5972, name=Thread-2173, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    6) Thread[id=5944, name=OverseerHdfsCoreFailoverThread-96018484180746243-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Throwable.fillInStackTrace(java.base@9-ea/Native Method)         at java.lang.Throwable.fillInStackTrace(java.base@9-ea/Throwable.java:788)         at java.lang.Throwable.<init>(java.base@9-ea/Throwable.java:271)         at java.lang.Exception.<init>(java.base@9-ea/Exception.java:66)         at java.lang.InterruptedException.<init>(java.base@9-ea/InterruptedException.java:67)         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=5977, name=zkCallback-1107-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    8) Thread[id=5976, name=zkCallback-1107-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=5971, name=OverseerHdfsCoreFailoverThread-96018505175793667-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   10) Thread[id=5966, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[49FB930F5CA93324]-SendThread(127.0.0.1:35001), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)   11) Thread[id=5975, name=zkCallback-1107-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=5978, name=zkCallback-1107-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   2) Thread[id=5939, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[49FB930F5CA93324]-SendThread(127.0.0.1:44025), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   3) Thread[id=5945, name=Thread-2171, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   4) Thread[id=5974, name=zkCallback-1107-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   5) Thread[id=5972, name=Thread-2173, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   6) Thread[id=5944, name=OverseerHdfsCoreFailoverThread-96018484180746243-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Throwable.fillInStackTrace(java.base@9-ea/Native Method)
        at java.lang.Throwable.fillInStackTrace(java.base@9-ea/Throwable.java:788)
        at java.lang.Throwable.<init>(java.base@9-ea/Throwable.java:271)
        at java.lang.Exception.<init>(java.base@9-ea/Exception.java:66)
        at java.lang.InterruptedException.<init>(java.base@9-ea/InterruptedException.java:67)
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   7) Thread[id=5977, name=zkCallback-1107-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   8) Thread[id=5976, name=zkCallback-1107-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=5971, name=OverseerHdfsCoreFailoverThread-96018505175793667-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  10) Thread[id=5966, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[49FB930F5CA93324]-SendThread(127.0.0.1:35001), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
  11) Thread[id=5975, name=zkCallback-1107-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324]:0)


FAILED:  org.apache.solr.cloud.AliasIntegrationTest.test

Error Message:
Error from server at http://127.0.0.1:45440/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:45440/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:C1AFACD5F2555EDC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.AliasIntegrationTest.test(AliasIntegrationTest.java:69)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
Error from server at http://127.0.0.1:44457: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore

Stack Trace:
java.lang.AssertionError: Error from server at http://127.0.0.1:44457: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore
	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:C1AFACD5F2555EDC]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:162)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:71)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:37553/_/n/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /_/n/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:37553/_/n/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /_/n/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:C1AFACD5F2555EDC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:152)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicZkTest.testBasic

Error Message:
No registered leader was found after waiting for 30000ms , collection: collection1 slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 30000ms , collection: collection1 slice: shard1
	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:E2018E1A8375B50A]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:718)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:685)
	at org.apache.solr.cloud.BasicZkTest.testBasic(BasicZkTest.java:51)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Error from server at http://127.0.0.1:46543/c/fs/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /c/fs/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:46543/c/fs/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /c/fs/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:C1AFACD5F2555EDC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:128)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
Error from server at https://127.0.0.1:37982/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:37982/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:C1AFACD5F2555EDC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.tryDelete(ChaosMonkeySafeLeaderTest.java:190)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:92)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration

Error Message:
Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: Timed out getting shard id for core: testcore

Stack Trace:
org.apache.solr.common.SolrException: Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: Timed out getting shard id for core: testcore
	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:F770F5A025D33D11]:0)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:785)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:718)
	at org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration(ClusterStateUpdateTest.java:158)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Unable to create core [testcore]
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:828)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:751)
	... 41 more
Caused by: org.apache.solr.common.SolrException: Timed out getting shard id for core: testcore
	at org.apache.solr.cloud.ZkController.waitForShardId(ZkController.java:1407)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1362)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	... 42 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:45973

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:45973
	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:C1AFACD5F2555EDC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:601)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:399)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testNoConfigSetExist(CollectionsAPIDistributedZkTest.java:601)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:176)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(java.base@9-ea/Native Method)
	at java.net.SocketInputStream.socketRead(java.base@9-ea/SocketInputStream.java:116)
	at java.net.SocketInputStream.read(java.base@9-ea/SocketInputStream.java:170)
	at java.net.SocketInputStream.read(java.base@9-ea/SocketInputStream.java:141)
	at sun.security.ssl.SSLSocketInputRecord.read(java.base@9-ea/SSLSocketInputRecord.java:425)
	at sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(java.base@9-ea/SSLSocketInputRecord.java:65)
	at sun.security.ssl.SSLSocketImpl.bytesInCompletePacket(java.base@9-ea/SSLSocketImpl.java:911)
	at sun.security.ssl.AppInputStream.read(java.base@9-ea/AppInputStream.java:144)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:495)
	... 47 more


FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Error from server at http://127.0.0.1:44444/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44444/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:C1AFACD5F2555EDC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
	at org.apache.solr.clou

[...truncated too long message...]

   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:244)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:241)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:241)
   [junit4]   2> 	at org.apache.solr.cloud.ElectionContext.cancelElection(ElectionContext.java:86)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContextBase.cancelElection(ElectionContext.java:143)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.cancelElection(ElectionContext.java:279)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:455)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(java.base@9-ea/Executors.java:514)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
   [junit4]   2> 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]   2> 5486983 INFO  (TEST-RecoveryZkTest.test-seed#[49FB930F5CA93324]) [    ] o.e.j.s.ServerConnector Stopped ServerConnector@528c5cb3{HTTP/1.1,[http/1.1]}{127.0.0.1:35106}
   [junit4]   2> 5486983 INFO  (TEST-RecoveryZkTest.test-seed#[49FB930F5CA93324]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1b210065{/lb/im,null,UNAVAILABLE}
   [junit4]   2> 5486984 INFO  (TEST-RecoveryZkTest.test-seed#[49FB930F5CA93324]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44398 44398
   [junit4]   2> 5487091 INFO  (Thread-2150) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44398 44398
   [junit4]   2> 5487092 WARN  (Thread-2150) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/aliases.json
   [junit4]   2> 	5	/solr/clusterprops.json
   [junit4]   2> 	4	/solr/security.json
   [junit4]   2> 	3	/solr/configs/conf1
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/clusterstate.json
   [junit4]   2> 	3	/solr/collections/collection1/state.json
   [junit4]   2> 	2	/solr/collections/collection1/leader_elect/shard1/election/96018520568037385-core_node1-n_0000000000
   [junit4]   2> 	2	/solr/overseer_elect/election/96018520568037385-127.0.0.1:33867_lb%2Fim-n_0000000001
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/collections
   [junit4]   2> 	4	/solr/live_nodes
   [junit4]   2> 	2	/solr/overseer/queue
   [junit4]   2> 	2	/solr/overseer/collection-queue-work
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryZkTest -Dtests.method=test -Dtests.seed=49FB930F5CA93324 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=sr-Cyrl -Dtests.timezone=Africa/Juba -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE  328s J0 | RecoveryZkTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:C1AFACD5F2555EDC]:0)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:127)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]   2> 5487093 INFO  (SUITE-RecoveryZkTest-seed#[49FB930F5CA93324]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest_49FB930F5CA93324-001
   [junit4]   2> NOTE: test params are: codec=HighCompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=15993, maxDocsPerChunk=1, blockSize=4), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=15993, blockSize=4)), sim=RandomSimilarity(queryNorm=true,coord=crazy): {}, locale=sr-Cyrl, timezone=Africa/Juba
   [junit4]   2> NOTE: Linux 4.4.0-21-generic amd64/Oracle Corporation 9-ea (64-bit)/cpus=12,threads=1,free=385556872,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [CacheHeaderTest, TestDefaultStatsCache, TestMergePolicyConfig, TestFieldCacheWithThreads, TestFieldResource, TestDynamicFieldCollectionResource, TestCustomDocTransformer, RequestLoggingTest, JSONWriterTest, TestDocSet, SharedFSAutoReplicaFailoverTest, TestRandomDVFaceting, TestClassicSimilarityFactory, XsltUpdateRequestHandlerTest, ConcurrentDeleteAndCreateCollectionTest, UUIDFieldTest, SmileWriterTest, TestBadConfig, TestLuceneMatchVersion, GraphQueryTest, TestConfigSetProperties, TestSmileRequest, TestLazyCores, TestRebalanceLeaders, TestOmitPositions, JavabinLoaderTest, TestHighlightDedupGrouping, TestManagedSchema, HdfsUnloadDistributedZkTest, SystemInfoHandlerTest, TestSizeLimitedDistributedMap, BasicZkTest, OutputWriterTest, CreateCollectionCleanupTest, MultiThreadedOCPTest, DistributedDebugComponentTest, TestReloadDeadlock, TestAnalyzedSuggestions, TestSubQueryTransformerCrossCore, CopyFieldTest, ZkNodePropsTest, FileBasedSpellCheckerTest, TestSolrQueryParserResource, TestExpandComponent, TestLeaderElectionZkExpiry, URLClassifyProcessorTest, TestUseDocValuesAsStored, TestManagedResourceStorage, TlogReplayBufferedWhileIndexingTest, QueryEqualityTest, WrapperMergePolicyFactoryTest, TestCustomSort, TestConfig, TestSolrConfigHandler, DeleteStatusTest, DisMaxRequestHandlerTest, BasicDistributedZkTest, TestSQLHandler, SolrRequestParserTest, TestManagedSynonymFilterFactory, ForceLeaderTest, DistanceUnitsTest, SSLMigrationTest, DistributedSuggestComponentTest, CdcrReplicationDistributedZkTest, ConjunctionSolrSpellCheckerTest, DeleteLastCustomShardedReplicaTest, TestBinaryField, TestOverriddenPrefixQueryForCustomFieldType, AlternateDirectoryTest, AnalysisErrorHandlingTest, PolyFieldTest, TestLRUCache, TestValueSourceCache, CoreAdminCreateDiscoverTest, TestXmlQParser, OverseerCollectionConfigSetProcessorTest, PrimUtilsTest, TestIBSimilarityFactory, HdfsDirectoryTest, PathHierarchyTokenizerFactoryTest, PreAnalyzedFieldTest, SuggestComponentContextFilterQueryTest, TestRTGBase, LeaderInitiatedRecoveryOnCommitTest, BlockJoinFacetDistribTest, TestDistribDocBasedVersion, OverseerTaskQueueTest, HdfsSyncSliceTest, InfoHandlerTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, HdfsRecoverLeaseTest, TestGraphTermsQParserPlugin, PingRequestHandlerTest, TestRuleBasedAuthorizationPlugin, DistribJoinFromCollectionTest, TestFaceting, BigEndianAscendingWordSerializerTest, TestCrossCoreJoin, VersionInfoTest, TestRemoteStreaming, TestFieldCache, ChaosMonkeySafeLeaderTest, SyncSliceTest, ShardRoutingTest, RecoveryZkTest]
   [junit4] Completed [580/613 (18!)] on J0 in 327.60s, 1 test, 1 failure <<< FAILURES!

[...truncated 161 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 2 suite-level errors, 15 errors, 4 failures, 1297 ignored (66 assumptions) [seed: 49FB930F5CA93324]

Total time: 160 minutes 33 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-6.x-Linux (64bit/jdk1.8.0_92) - Build # 832 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/832/
Java: 64bit/jdk1.8.0_92 -XX:+UseCompressedOops -XX:+UseSerialGC

22 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
17 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest:     1) Thread[id=96583, name=Thread-2487, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    2) Thread[id=96610, name=Thread-2489, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    3) Thread[id=96607, name=OverseerStateUpdate-96017324924534787-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)         at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=96614, name=zkCallback-25999-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=96580, name=OverseerStateUpdate-96017303941742595-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=96577, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[1D3661B79789AE89]-SendThread(127.0.0.1:33290), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    7) Thread[id=96605, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[1D3661B79789AE89]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    8) Thread[id=96608, name=OverseerCollectionConfigSetProcessor-96017324924534787-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=96616, name=zkCallback-25999-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=96578, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[1D3661B79789AE89]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   11) Thread[id=96615, name=zkCallback-25999-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   12) Thread[id=96612, name=zkCallback-25999-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   13) Thread[id=96609, name=OverseerHdfsCoreFailoverThread-96017324924534787-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)   14) Thread[id=96604, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[1D3661B79789AE89]-SendThread(127.0.0.1:43075), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   15) Thread[id=96613, name=zkCallback-25999-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   16) Thread[id=96582, name=OverseerHdfsCoreFailoverThread-96017303941742595-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)   17) Thread[id=96581, name=OverseerCollectionConfigSetProcessor-96017303941742595-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 17 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest: 
   1) Thread[id=96583, name=Thread-2487, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   2) Thread[id=96610, name=Thread-2489, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   3) Thread[id=96607, name=OverseerStateUpdate-96017324924534787-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
        at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)
        at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)
        at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=96614, name=zkCallback-25999-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=96580, name=OverseerStateUpdate-96017303941742595-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=96577, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[1D3661B79789AE89]-SendThread(127.0.0.1:33290), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   7) Thread[id=96605, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[1D3661B79789AE89]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   8) Thread[id=96608, name=OverseerCollectionConfigSetProcessor-96017324924534787-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=96616, name=zkCallback-25999-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=96578, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[1D3661B79789AE89]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  11) Thread[id=96615, name=zkCallback-25999-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  12) Thread[id=96612, name=zkCallback-25999-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  13) Thread[id=96609, name=OverseerHdfsCoreFailoverThread-96017324924534787-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
  14) Thread[id=96604, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[1D3661B79789AE89]-SendThread(127.0.0.1:43075), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  15) Thread[id=96613, name=zkCallback-25999-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  16) Thread[id=96582, name=OverseerHdfsCoreFailoverThread-96017303941742595-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
  17) Thread[id=96581, name=OverseerCollectionConfigSetProcessor-96017303941742595-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([1D3661B79789AE89]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=96583, name=Thread-2487, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    2) Thread[id=96610, name=Thread-2489, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    3) Thread[id=96614, name=zkCallback-25999-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=96577, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[1D3661B79789AE89]-SendThread(127.0.0.1:33290), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    5) Thread[id=96582, name=OverseerHdfsCoreFailoverThread-96017303941742595-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=96616, name=zkCallback-25999-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=96615, name=zkCallback-25999-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=96612, name=zkCallback-25999-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=96609, name=OverseerHdfsCoreFailoverThread-96017324924534787-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=96604, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[1D3661B79789AE89]-SendThread(127.0.0.1:43075), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)   11) Thread[id=96613, name=zkCallback-25999-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=96583, name=Thread-2487, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   2) Thread[id=96610, name=Thread-2489, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   3) Thread[id=96614, name=zkCallback-25999-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=96577, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[1D3661B79789AE89]-SendThread(127.0.0.1:33290), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   5) Thread[id=96582, name=OverseerHdfsCoreFailoverThread-96017303941742595-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=96616, name=zkCallback-25999-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=96615, name=zkCallback-25999-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=96612, name=zkCallback-25999-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=96609, name=OverseerHdfsCoreFailoverThread-96017324924534787-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=96604, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[1D3661B79789AE89]-SendThread(127.0.0.1:43075), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
  11) Thread[id=96613, name=zkCallback-25999-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([1D3661B79789AE89]:0)


FAILED:  org.apache.solr.cloud.AsyncMigrateRouteKeyTest.test

Error Message:
Error from server at http://127.0.0.1:34524/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:34524/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([1D3661B79789AE89:95625E6D3975C371]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest(MigrateRouteKeyTest.java:135)
	at org.apache.solr.cloud.AsyncMigrateRouteKeyTest.test(AsyncMigrateRouteKeyTest.java:44)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
Error from server at http://127.0.0.1:34463/_fgc/d: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore

Stack Trace:
java.lang.AssertionError: Error from server at http://127.0.0.1:34463/_fgc/d: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore
	at __randomizedtesting.SeedInfo.seed([1D3661B79789AE89:95625E6D3975C371]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:162)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:71)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:34495/_/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /_/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:34495/_/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /_/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([1D3661B79789AE89:95625E6D3975C371]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:152)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.BasicZkTest.testBasic

Error Message:
No registered leader was found after waiting for 30000ms , collection: collection1 slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 30000ms , collection: collection1 slice: shard1
	at __randomizedtesting.SeedInfo.seed([1D3661B79789AE89:B6CC7CA2485528A7]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:718)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:685)
	at org.apache.solr.cloud.BasicZkTest.testBasic(BasicZkTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
Error from server at http://127.0.0.1:39777/jg_tvr/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /jg_tvr/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:39777/jg_tvr/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /jg_tvr/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([1D3661B79789AE89:95625E6D3975C371]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.tryDelete(ChaosMonkeySafeLeaderTest.java:190)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:92)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration

Error Message:
Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: Timed out getting shard id for core: testcore

Stack Trace:
org.apache.solr.common.SolrException: Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: Timed out getting shard id for core: testcore
	at __randomizedtesting.SeedInfo.seed([1D3661B79789AE89:A3BD0718EEF3A0BC]:0)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:785)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:718)
	at org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration(ClusterStateUpdateTest.java:158)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: Unable to create core [testcore]
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:828)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:751)
	... 41 more
Caused by: org.apache.solr.common.SolrException: Timed out getting shard id for core: testcore
	at org.apache.solr.cloud.ZkController.waitForShardId(ZkController.java:1407)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1362)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	... 42 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:44459/kh_/s

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:44459/kh_/s
	at __randomizedtesting.SeedInfo.seed([1D3661B79789AE89:95625E6D3975C371]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:601)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:399)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testNoConfigSetExist(CollectionsAPIDistributedZkTest.java:601)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDis

[...truncated too long message...]

deName for collection1
   [junit4]    > 	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
   [junit4]    > 	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
   [junit4]    > 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
   [junit4]    > 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]    > 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
   [junit4]    > 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]    > 	at org.eclipse.jetty.server.Server.handle(Server.java:518)
   [junit4]    > 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
   [junit4]    > 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
   [junit4]    > 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]    > 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]    > 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]    > 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
   [junit4]    > 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
   [junit4]    > 	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    > 	... 1 more
   [junit4]    > ,code=500}</pre></p>
   [junit4]    > <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
   [junit4]    > </body>
   [junit4]    > </html>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([1D3661B79789AE89:95625E6D3975C371]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.indexDoc(AbstractFullDistribZkTestBase.java:775)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.index(BaseDistributedSearchTestCase.java:483)
   [junit4]    > 	at org.apache.solr.cloud.TriLevelCompositeIdRoutingTest.doAddDoc(TriLevelCompositeIdRoutingTest.java:142)
   [junit4]    > 	at org.apache.solr.cloud.TriLevelCompositeIdRoutingTest.doTriLevelHashingTest(TriLevelCompositeIdRoutingTest.java:90)
   [junit4]    > 	at org.apache.solr.cloud.TriLevelCompositeIdRoutingTest.test(TriLevelCompositeIdRoutingTest.java:69)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 10113349 INFO  (SUITE-TriLevelCompositeIdRoutingTest-seed#[1D3661B79789AE89]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.TriLevelCompositeIdRoutingTest_1D3661B79789AE89-001
   [junit4]   2> Jun 05, 2016 7:28:59 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene60), sim=ClassicSimilarity, locale=sv-SE, timezone=Etc/GMT-9
   [junit4]   2> NOTE: Linux 4.4.0-21-generic amd64/Oracle Corporation 1.8.0_92 (64-bit)/cpus=12,threads=1,free=179389160,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [TestReplicaProperties, ExitableDirectoryReaderTest, DistribCursorPagingTest, BlockDirectoryTest, TestHashQParserPlugin, SpatialFilterTest, SortByFunctionTest, TestAuthorizationFramework, ZkCLITest, TestCopyFieldCollectionResource, TermVectorComponentTest, DOMUtilTest, TestScoreJoinQPNoScore, TestIntervalFaceting, TestReloadAndDeleteDocs, WordBreakSolrSpellCheckerTest, TestLFUCache, TestUpdate, TestSolrConfigHandlerCloud, LoggingHandlerTest, TestJsonFacets, TestPivotHelperCode, TestSerializedLuceneMatchVersion, TestConfigSetsAPIZkFailure, TestGeoJSONResponseWriter, HttpPartitionTest, BlockJoinFacetRandomTest, TestRestManager, HdfsRecoveryZkTest, AsyncMigrateRouteKeyTest, DeleteReplicaTest, SpellCheckCollatorTest, BitVectorTest, TestSolrDynamicMBean, CdcrReplicationHandlerTest, TestSolrCoreProperties, HdfsLockFactoryTest, FastVectorHighlighterTest, TestEmbeddedSolrServerConstructors, ParsingFieldUpdateProcessorsTest, TestSha256AuthenticationProvider, OverseerRolesTest, DistributedExpandComponentTest, HdfsCollectionsAPIDistributedZkTest, SuggestComponentTest, HdfsBasicDistributedZkTest, TestCloudSchemaless, SoftAutoCommitTest, TestManagedSchemaAPI, CollectionsAPIDistributedZkTest, DistanceFunctionTest, TestSimpleTrackingShardHandler, TestSortByMinMaxFunction, TestQueryUtils, CustomCollectionTest, OpenExchangeRatesOrgProviderTest, TestTestInjection, FileUtilsTest, HighlighterMaxOffsetTest, NumericFieldsTest, DocumentBuilderTest, ZkControllerTest, HdfsWriteToMultipleCollectionsTest, TestChildDocTransformer, HdfsThreadLeakTest, TestExceedMaxTermLength, UniqFieldsUpdateProcessorFactoryTest, TestQueryTypes, ClusterStateUpdateTest, TestConfigOverlay, TestFreeTextSuggestions, ShardRoutingTest, TestSubQueryTransformerDistrib, CollectionReloadTest, SolrXmlInZkTest, TestDocumentBuilder, SolrPluginUtilsTest, UpdateRequestProcessorFactoryTest, TestCollapseQParserPlugin, TestSolrDeletionPolicy2, TestCursorMarkWithoutUniqueKey, TestPerFieldSimilarityWithDefaultOverride, ConvertedLegacyTest, ShardRoutingCustomTest, OpenCloseCoreStressTest, TestInfoStreamLogging, HLLSerializationTest, TestDistribIDF, TestWordDelimiterFilterFactory, TriLevelCompositeIdRoutingTest]
   [junit4] Completed [602/613 (19!)] on J1 in 612.41s, 1 test, 1 error <<< FAILURES!

[...truncated 53 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 2 suite-level errors, 16 errors, 4 failures, 1294 ignored (69 assumptions) [seed: 1D3661B79789AE89]

Total time: 192 minutes 55 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-6.x-Linux (64bit/jdk1.8.0_92) - Build # 831 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/831/
Java: 64bit/jdk1.8.0_92 -XX:+UseCompressedOops -XX:+UseParallelGC

15 tests failed.
FAILED:  org.apache.solr.schema.TestBulkSchemaConcurrent.test

Error Message:
[[["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], ["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], [   {     "delete-copy-field":{       "source":"a0",       "dest":["hello_lol0"]},     "errorMessages":["Copy field directive not found: 'a0' -> 'hello_lol0'\n"]},   {     "delete-field":{"name":"a0"},     "errorMessages":["The field 'a0' is not present in this schema, and so cannot be deleted.\n"]},   {     "delete-dynamic-field":{"name":"*_lol0"},     "errorMessages":["The dynamic field '*_lol0' is not present in this schema, and so cannot be deleted.\n"]},   {     "delete-field-type":{"name":"mystr0"},     "errorMessages":["The field type 'mystr0' is not present in this schema, and so cannot be deleted.\n"]}]], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], ["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], ["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"]], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], ["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], [   {     "delete-copy-field":{       "source":"a2",       "dest":["hello_lol2"]},     "errorMessages":["Copy field directive not found: 'a2' -> 'hello_lol2'\n"]},   {     "delete-field":{"name":"a2"},     "errorMessages":["The field 'a2' is not present in this schema, and so cannot be deleted.\n"]},   {     "delete-dynamic-field":{"name":"*_lol2"},     "errorMessages":["The dynamic field '*_lol2' is not present in this schema, and so cannot be deleted.\n"]},   {     "delete-field-type":{"name":"mystr2"},     "errorMessages":["The field type 'mystr2' is not present in this schema, and so cannot be deleted.\n"]}]], [], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], [   {     "replace-field":{       "name":"a4",       "type":"text",       "stored":true,       "indexed":true},     "errorMessages":["The field 'a4' is not present in this schema, and so cannot be replaced.\n"]},   {     "replace-dynamic-field":{       "name":"*_lol4",       "type":"text",       "stored":true,       "indexed":true},     "errorMessages":["The dynamic field '*_lol4' is not present in this schema, and so cannot be replaced.\n"]},   {     "replace-field-type":{       "name":"mystr4",       "class":"solr.TextField"},     "errorMessages":["The field type 'mystr4' is not present in this schema, and so cannot be replaced.\n"]}], [   {     "delete-copy-field":{       "source":"a4",       "dest":["hello_lol4"]},     "errorMessages":["Copy field directive not found: 'a4' -> 'hello_lol4'\n"]},   {     "delete-field":{"name":"a4"},     "errorMessages":["The field 'a4' is not present in this schema, and so cannot be deleted.\n"]},   {     "delete-dynamic-field":{"name":"*_lol4"},     "errorMessages":["The dynamic field '*_lol4' is not present in this schema, and so cannot be deleted.\n"]},   {     "delete-field-type":{"name":"mystr4"},     "errorMessages":["The field type 'mystr4' is not present in this schema, and so cannot be deleted.\n"]}]]]

Stack Trace:
java.lang.AssertionError: [[["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], ["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], [
  {
    "delete-copy-field":{
      "source":"a0",
      "dest":["hello_lol0"]},
    "errorMessages":["Copy field directive not found: 'a0' -> 'hello_lol0'\n"]},
  {
    "delete-field":{"name":"a0"},
    "errorMessages":["The field 'a0' is not present in this schema, and so cannot be deleted.\n"]},
  {
    "delete-dynamic-field":{"name":"*_lol0"},
    "errorMessages":["The dynamic field '*_lol0' is not present in this schema, and so cannot be deleted.\n"]},
  {
    "delete-field-type":{"name":"mystr0"},
    "errorMessages":["The field type 'mystr0' is not present in this schema, and so cannot be deleted.\n"]}]], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], ["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], ["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"]], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], ["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], [
  {
    "delete-copy-field":{
      "source":"a2",
      "dest":["hello_lol2"]},
    "errorMessages":["Copy field directive not found: 'a2' -> 'hello_lol2'\n"]},
  {
    "delete-field":{"name":"a2"},
    "errorMessages":["The field 'a2' is not present in this schema, and so cannot be deleted.\n"]},
  {
    "delete-dynamic-field":{"name":"*_lol2"},
    "errorMessages":["The dynamic field '*_lol2' is not present in this schema, and so cannot be deleted.\n"]},
  {
    "delete-field-type":{"name":"mystr2"},
    "errorMessages":["The field type 'mystr2' is not present in this schema, and so cannot be deleted.\n"]}]], [], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], [
  {
    "replace-field":{
      "name":"a4",
      "type":"text",
      "stored":true,
      "indexed":true},
    "errorMessages":["The field 'a4' is not present in this schema, and so cannot be replaced.\n"]},
  {
    "replace-dynamic-field":{
      "name":"*_lol4",
      "type":"text",
      "stored":true,
      "indexed":true},
    "errorMessages":["The dynamic field '*_lol4' is not present in this schema, and so cannot be replaced.\n"]},
  {
    "replace-field-type":{
      "name":"mystr4",
      "class":"solr.TextField"},
    "errorMessages":["The field type 'mystr4' is not present in this schema, and so cannot be replaced.\n"]}], [
  {
    "delete-copy-field":{
      "source":"a4",
      "dest":["hello_lol4"]},
    "errorMessages":["Copy field directive not found: 'a4' -> 'hello_lol4'\n"]},
  {
    "delete-field":{"name":"a4"},
    "errorMessages":["The field 'a4' is not present in this schema, and so cannot be deleted.\n"]},
  {
    "delete-dynamic-field":{"name":"*_lol4"},
    "errorMessages":["The dynamic field '*_lol4' is not present in this schema, and so cannot be deleted.\n"]},
  {
    "delete-field-type":{"name":"mystr4"},
    "errorMessages":["The field type 'mystr4' is not present in this schema, and so cannot be deleted.\n"]}]]]
	at __randomizedtesting.SeedInfo.seed([77DB098E5818E21:8F298F424B7DE3D9]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.schema.TestBulkSchemaConcurrent.test(TestBulkSchemaConcurrent.java:115)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.schema.TestCloudSchemaless.test

Error Message:
QUERY FAILED: xpath=/response/arr[@name='fields']/lst/str[@name='name'][.='newTestFieldInt0']  request=/schema/fields?wt=xml  response=<?xml version="1.0" encoding="UTF-8"?> <response> <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><arr name="fields"><lst><str name="name">_root_</str><str name="type">string</str><bool name="multiValued">false</bool><bool name="indexed">true</bool><bool name="stored">true</bool></lst><lst><str name="name">_version_</str><str name="type">long</str><bool name="indexed">true</bool><bool name="stored">true</bool></lst><lst><str name="name">constantField</str><str name="type">tdouble</str></lst><lst><str name="name">id</str><str name="type">string</str><bool name="multiValued">false</bool><bool name="indexed">true</bool><bool name="required">true</bool><bool name="stored">true</bool></lst><lst><str name="name">newTestFieldInt1</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt10</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt100</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt101</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt102</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt103</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt104</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt105</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt106</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt107</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt108</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt109</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt11</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt110</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt111</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt112</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt113</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt114</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt115</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt116</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt117</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt118</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt119</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt12</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt120</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt121</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt122</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt123</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt124</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt125</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt126</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt127</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt128</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt129</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt13</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt130</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt131</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt132</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt133</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt134</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt135</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt136</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt137</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt138</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt139</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt14</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt140</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt141</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt142</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt143</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt144</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt145</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt146</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt147</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt148</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt149</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt15</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt150</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt151</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt152</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt153</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt154</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt155</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt156</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt157</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt158</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt159</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt16</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt160</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt161</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt162</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt163</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt164</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt165</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt166</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt167</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt168</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt169</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt17</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt170</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt171</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt172</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt173</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt174</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt175</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt176</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt177</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt178</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt179</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt18</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt180</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt181</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt182</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt183</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt184</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt185</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt186</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt187</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt188</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt189</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt19</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt190</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt191</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt192</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt193</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt194</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt195</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt196</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt197</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt198</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt199</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt2</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt20</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt200</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt201</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt202</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt203</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt204</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt205</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt206</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt207</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt208</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt209</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt21</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt210</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt211</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt212</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt213</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt214</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt215</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt216</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt217</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt218</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt219</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt22</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt220</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt221</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt222</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt223</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt224</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt225</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt226</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt227</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt228</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt229</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt23</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt230</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt231</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt232</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt233</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt234</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt235</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt236</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt237</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt238</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt239</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt24</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt240</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt241</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt242</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt243</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt244</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt245</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt246</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt247</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt248</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt249</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt25</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt250</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt251</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt252</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt253</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt254</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt255</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt256</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt257</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt258</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt259</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt26</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt260</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt261</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt262</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt263</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt264</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt265</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt266</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt267</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt268</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt269</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt27</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt270</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt271</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt272</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt273</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt274</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt275</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt276</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt277</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt278</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt279</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt28</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt280</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt281</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt282</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt283</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt284</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt285</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt286</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt287</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt288</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt289</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt29</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt290</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt291</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt292</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt293</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt294</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt295</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt296</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt297</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt298</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt299</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt3</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt30</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt300</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt301</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt302</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt303</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt304</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt305</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt306</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt307</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt308</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt309</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt31</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt310</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt311</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt312</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt313</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt314</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt315</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt316</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt317</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt318</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt319</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt32</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt320</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt321</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt322</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt323</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt324</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt325</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt326</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt327</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt328</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt329</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt33</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt330</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt331</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt332</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt333</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt334</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt335</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt336</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt337</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt338</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt339</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt34</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt340</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt341</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt342</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt343</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt344</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt345</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt346</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt347</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt348</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt349</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt35</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt350</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt351</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt352</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt353</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt354</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt355</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt356</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt357</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt358</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt359</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt36</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt360</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt361</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt362</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt363</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt364</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt365</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt366</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt367</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt368</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt369</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt37</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt370</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt371</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt372</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt373</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt374</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt375</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt376</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt377</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt378</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt379</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt38</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt380</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt381</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt382</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt383</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt384</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt385</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt386</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt387</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt388</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt389</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt39</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt390</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt391</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt392</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt393</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt394</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt395</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt396</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt397</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt398</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt399</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt4</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt40</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt400</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt401</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt402</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt403</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt404</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt405</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt406</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt407</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt408</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt409</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt41</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt410</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt411</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt412</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt413</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt414</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt415</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt416</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt417</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt418</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt419</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt42</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt420</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt421</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt422</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt423</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt424</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt425</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt426</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt427</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt428</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt429</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt43</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt430</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt431</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt432</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt433</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt434</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt435</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt436</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt437</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt438</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt439</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt44</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt440</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt441</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt442</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt443</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt444</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt445</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt446</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt447</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt448</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt449</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt45</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt46</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt47</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt48</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt49</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt5</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt50</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt51</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt52</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt53</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt54</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt55</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt56</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt57</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt58</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt59</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt6</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt60</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt61</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt62</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt63</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt64</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt65</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt66</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt67</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt68</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt69</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt7</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt70</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt71</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt72</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt73</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt74</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt75</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt76</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt77</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt78</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt79</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt8</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt80</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt81</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt82</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt83</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt84</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt85</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt86</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt87</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt88</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt89</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt9</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt90</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt91</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt92</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt93</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt94</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt95</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt96</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt97</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt98</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt99</str><str name="type">tlong</str></lst></arr> </response> 

Stack Trace:
java.lang.AssertionError: QUERY FAILED: xpath=/response/arr[@name='fields']/lst/str[@name='name'][.='newTestFieldInt0']  request=/schema/fields?wt=xml  response=<?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><arr name="fields"><lst><str name="name">_root_</str><str name="type">string</str><bool name="multiValued">false</bool><bool name="indexed">true</bool><bool name="stored">true</bool></lst><lst><str name="name">_version_</str><str name="type">long</str><bool name="indexed">true</bool><bool name="stored">true</bool></lst><lst><str name="name">constantField</str><str name="type">tdouble</str></lst><lst><str name="name">id</str><str name="type">string</str><bool name="multiValued">false</bool><bool name="indexed">true</bool><bool name="required">true</bool><bool name="stored">true</bool></lst><lst><str name="name">newTestFieldInt1</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt10</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt100</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt101</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt102</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt103</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt104</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt105</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt106</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt107</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt108</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt109</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt11</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt110</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt111</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt112</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt113</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt114</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt115</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt116</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt117</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt118</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt119</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt12</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt120</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt121</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt122</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt123</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt124</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt125</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt126</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt127</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt128</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt129</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt13</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt130</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt131</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt132</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt133</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt134</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt135</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt136</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt137</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt138</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt139</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt14</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt140</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt141</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt142</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt143</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt144</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt145</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt146</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt147</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt148</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt149</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt15</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt150</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt151</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt152</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt153</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt154</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt155</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt156</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt157</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt158</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt159</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt16</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt160</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt161</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt162</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt163</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt164</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt165</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt166</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt167</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt168</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt169</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt17</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt170</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt171</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt172</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt173</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt174</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt175</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt176</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt177</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt178</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt179</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt18</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt180</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt181</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt182</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt183</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt184</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt185</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt186</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt187</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt188</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt189</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt19</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt190</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt191</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt192</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt193</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt194</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt195</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt196</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt197</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt198</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt199</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt2</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt20</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt200</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt201</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt202</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt203</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt204</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt205</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt206</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt207</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt208</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt209</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt21</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt210</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt211</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt212</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt213</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt214</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt215</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt216</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt217</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt218</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt219</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt22</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt220</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt221</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt222</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt223</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt224</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt225</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt226</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt227</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt228</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt229</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt23</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt230</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt231</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt232</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt233</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt234</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt235</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt236</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt237</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt238</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt239</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt24</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt240</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt241</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt242</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt243</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt244</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt245</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt246</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt247</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt248</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt249</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt25</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt250</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt251</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt252</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt253</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt254</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt255</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt256</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt257</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt258</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt259</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt26</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt260</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt261</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt262</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt263</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt264</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt265</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt266</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt267</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt268</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt269</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt27</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt270</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt271</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt272</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt273</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt274</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt275</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt276</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt277</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt278</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt279</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt28</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt280</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt281</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt282</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt283</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt284</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt285</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt286</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt287</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt288</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt289</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt29</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt290</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt291</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt292</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt293</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt294</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt295</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt296</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt297</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt298</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt299</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt3</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt30</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt300</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt301</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt302</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt303</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt304</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt305</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt306</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt307</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt308</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt309</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt31</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt310</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt311</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt312</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt313</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt314</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt315</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt316</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt317</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt318</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt319</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt32</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt320</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt321</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt322</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt323</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt324</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt325</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt326</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt327</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt328</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt329</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt33</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt330</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt331</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt332</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt333</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt334</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt335</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt336</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt337</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt338</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt339</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt34</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt340</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt341</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt342</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt343</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt344</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt345</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt346</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt347</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt348</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt349</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt35</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt350</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt351</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt352</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt353</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt354</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt355</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt356</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt357</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt358</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt359</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt36</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt360</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt361</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt362</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt363</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt364</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt365</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt366</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt367</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt368</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt369</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt37</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt370</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt371</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt372</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt373</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt374</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt375</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt376</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt377</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt378</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt379</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt38</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt380</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt381</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt382</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt383</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt384</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt385</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt386</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt387</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt388</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt389</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt39</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt390</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt391</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt392</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt393</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt394</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt395</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt396</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt397</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt398</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt399</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt4</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt40</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt400</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt401</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt402</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt403</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt404</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt405</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt406</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt407</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt408</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt409</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt41</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt410</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt411</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt412</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt413</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt414</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt415</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt416</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt417</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt418</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt419</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt42</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt420</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt421</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt422</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt423</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt424</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt425</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt426</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt427</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt428</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt429</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt43</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt430</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt431</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt432</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt433</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt434</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt435</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt436</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt437</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt438</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt439</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt44</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt440</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt441</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt442</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt443</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt444</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt445</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt446</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt447</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt448</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt449</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt45</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt46</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt47</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt48</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt49</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt5</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt50</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt51</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt52</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt53</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt54</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt55</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt56</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt57</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt58</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt59</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt6</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt60</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt61</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt62</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt63</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt64</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt65</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt66</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt67</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt68</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt69</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt7</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt70</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt71</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt72</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt73</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt74</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt75</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt76</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt77</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt78</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt79</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt8</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt80</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt81</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt82</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt83</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt84</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt85</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt86</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt87</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt88</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt89</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt9</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt90</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt91</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt92</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt93</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt94</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt95</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt96</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt97</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt98</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt99</str><str name="type">tlong</str></lst></arr>
</response>

	at __randomizedtesting.SeedInfo.seed([77DB098E5818E21:8F298F424B7DE3D9]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.schema.TestCloudSchemaless.test(TestCloudSchemaless.java:139)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtest

[...truncated too long message...]

re.CoreContainer.getCore(CoreContainer.java:1080)
   [junit4]    > 	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
   [junit4]    > 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
   [junit4]    > 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]    > 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
   [junit4]    > 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]    > 	at org.eclipse.jetty.server.Server.handle(Server.java:518)
   [junit4]    > 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
   [junit4]    > 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
   [junit4]    > 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]    > 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]    > 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]    > 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
   [junit4]    > 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
   [junit4]    > 	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    > 	... 1 more
   [junit4]    > ,code=500}</pre></p>
   [junit4]    > <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
   [junit4]    > </body>
   [junit4]    > </html>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([77DB098E5818E21:8F298F424B7DE3D9]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
   [junit4]    > 	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:128)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 7517724 INFO  (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[77DB098E5818E21]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_77DB098E5818E21-001
   [junit4]   2> Jun 04, 2016 11:42:27 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene60): {}, docValues:{}, maxPointsInLeafNode=1497, maxMBSortInHeap=5.193530687732812, sim=RandomSimilarity(queryNorm=false,coord=crazy): {}, locale=no, timezone=Indian/Mayotte
   [junit4]   2> NOTE: Linux 4.4.0-21-generic amd64/Oracle Corporation 1.8.0_92 (64-bit)/cpus=12,threads=1,free=241266472,total=527958016
   [junit4]   2> NOTE: All tests run in this JVM: [SolrPluginUtilsTest, TestUninvertingReader, HLLSerializationTest, DistanceFunctionTest, TestHashPartitioner, ConfigSetsAPITest, TestLRUStatsCache, CSVRequestHandlerTest, FileUtilsTest, TestConfigOverlay, AsyncMigrateRouteKeyTest, TestSolrCloudWithKerberosAlt, SolrIndexSplitterTest, TestFieldTypeCollectionResource, TestCloudSchemaless, TestWordDelimiterFilterFactory, RuleEngineTest, SecurityConfHandlerTest, DistributedQueueTest, TestRawTransformer, BlockJoinFacetRandomTest, TimeZoneUtilsTest, TestDownShardTolerantSearch, CdcrUpdateLogTest, OpenCloseCoreStressTest, TestDFRSimilarityFactory, DistributedFacetPivotSmallAdvancedTest, CheckHdfsIndexTest, TestQuerySenderNoQuery, SortByFunctionTest, TestSolrConfigHandlerCloud, TestConfigSets, BadCopyFieldTest, TestGeoJSONResponseWriter, AliasIntegrationTest, TestMiniSolrCloudClusterBase, TestEmbeddedSolrServerConstructors, TestHashQParserPlugin, HighlighterMaxOffsetTest, TestCursorMarkWithoutUniqueKey, ParsingFieldUpdateProcessorsTest, TestSolrDynamicMBean, TestLFUCache, TermsComponentTest, TestInfoStreamLogging, TestRestManager, SolrCmdDistributorTest, UniqFieldsUpdateProcessorFactoryTest, TestUpdate, TestSortByMinMaxFunction, CustomCollectionTest, CollectionReloadTest, TestStressVersions, TestSerializedLuceneMatchVersion, TestCoreContainer, TestDistribIDF, CollectionsAPIDistributedZkTest, TestSchemaResource, DocValuesMissingTest, TestBulkSchemaConcurrent, TestDynamicFieldResource, TolerantUpdateProcessorTest, HighlighterTest, TestFieldCacheReopen, SortSpecParsingTest, TestCollapseQParserPlugin, HdfsThreadLeakTest, HdfsWriteToMultipleCollectionsTest, SpellCheckCollatorTest, IndexBasedSpellCheckerTest, ExitableDirectoryReaderTest, BlockDirectoryTest, ClusterStateUpdateTest, AddSchemaFieldsUpdateProcessorFactoryTest, DistributedIntervalFacetingTest, BasicFunctionalityTest, TestBM25SimilarityFactory, DOMUtilTest, TestSolrCoreProperties, TestManagedSchemaAPI, OutOfBoxZkACLAndCredentialsProvidersTest, TestSSLRandomization, DeleteReplicaTest, DistributedExpandComponentTest, TestObjectReleaseTracker, TestTestInjection, RestartWhileUpdatingTest, BitVectorTest, LeaderInitiatedRecoveryOnShardRestartTest, FacetPivotSmallTest, TestIntervalFaceting, ZkStateReaderTest, TestFreeTextSuggestions, OpenExchangeRatesOrgProviderTest, ChaosMonkeyNothingIsSafeTest]
   [junit4] Completed [612/613 (12!)] on J1 in 613.25s, 1 test, 1 error <<< FAILURES!

[...truncated 23 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 2 suite-level errors, 9 errors, 4 failures, 1580 ignored (66 assumptions) [seed: 77DB098E5818E21]

Total time: 144 minutes 59 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS-EA] Lucene-Solr-6.x-Linux (32bit/jdk-9-ea+121) - Build # 830 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/830/
Java: 32bit/jdk-9-ea+121 -client -XX:+UseG1GC

15 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest:     1) Thread[id=6445, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[F65295FADAA2F5CD]-SendThread(127.0.0.1:46374), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    2) Thread[id=6454, name=zkCallback-1200-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    3) Thread[id=6420, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[F65295FADAA2F5CD]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    4) Thread[id=6456, name=zkCallback-1200-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    5) Thread[id=6422, name=OverseerStateUpdate-96014251173085187-127.0.0.1:7000_solr-n_0000000000, state=WAITING, group=Overseer state updater.]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342)         at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153)         at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348)         at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    6) Thread[id=6453, name=zkCallback-1200-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=6425, name=Thread-2339, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    8) Thread[id=6455, name=zkCallback-1200-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=6451, name=Thread-2341, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)   10) Thread[id=6423, name=OverseerCollectionConfigSetProcessor-96014251173085187-127.0.0.1:7000_solr-n_0000000000, state=WAITING, group=Overseer collection creation process.]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342)         at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153)         at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348)         at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   11) Thread[id=6448, name=OverseerStateUpdate-96014272158892035-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)         at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   12) Thread[id=6450, name=OverseerHdfsCoreFailoverThread-96014272158892035-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   13) Thread[id=6419, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[F65295FADAA2F5CD]-SendThread(127.0.0.1:41542), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   14) Thread[id=6446, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[F65295FADAA2F5CD]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   15) Thread[id=6449, name=OverseerCollectionConfigSetProcessor-96014272158892035-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   16) Thread[id=6424, name=OverseerHdfsCoreFailoverThread-96014251173085187-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest: 
   1) Thread[id=6445, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[F65295FADAA2F5CD]-SendThread(127.0.0.1:46374), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   2) Thread[id=6454, name=zkCallback-1200-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   3) Thread[id=6420, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[F65295FADAA2F5CD]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   4) Thread[id=6456, name=zkCallback-1200-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   5) Thread[id=6422, name=OverseerStateUpdate-96014251173085187-127.0.0.1:7000_solr-n_0000000000, state=WAITING, group=Overseer state updater.]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   6) Thread[id=6453, name=zkCallback-1200-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   7) Thread[id=6425, name=Thread-2339, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   8) Thread[id=6455, name=zkCallback-1200-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=6451, name=Thread-2341, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
  10) Thread[id=6423, name=OverseerCollectionConfigSetProcessor-96014251173085187-127.0.0.1:7000_solr-n_0000000000, state=WAITING, group=Overseer collection creation process.]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  11) Thread[id=6448, name=OverseerStateUpdate-96014272158892035-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
        at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)
        at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)
        at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  12) Thread[id=6450, name=OverseerHdfsCoreFailoverThread-96014272158892035-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  13) Thread[id=6419, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[F65295FADAA2F5CD]-SendThread(127.0.0.1:41542), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  14) Thread[id=6446, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[F65295FADAA2F5CD]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  15) Thread[id=6449, name=OverseerCollectionConfigSetProcessor-96014272158892035-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  16) Thread[id=6424, name=OverseerHdfsCoreFailoverThread-96014251173085187-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
	at __randomizedtesting.SeedInfo.seed([F65295FADAA2F5CD]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=6445, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[F65295FADAA2F5CD]-SendThread(127.0.0.1:46374), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    2) Thread[id=6454, name=zkCallback-1200-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    3) Thread[id=6456, name=zkCallback-1200-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    4) Thread[id=6450, name=OverseerHdfsCoreFailoverThread-96014272158892035-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Throwable.fillInStackTrace(java.base@9-ea/Native Method)         at java.lang.Throwable.fillInStackTrace(java.base@9-ea/Throwable.java:788)         at java.lang.Throwable.<init>(java.base@9-ea/Throwable.java:271)         at java.lang.Exception.<init>(java.base@9-ea/Exception.java:66)         at java.lang.InterruptedException.<init>(java.base@9-ea/InterruptedException.java:67)         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    5) Thread[id=6453, name=zkCallback-1200-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    6) Thread[id=6419, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[F65295FADAA2F5CD]-SendThread(127.0.0.1:41542), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    7) Thread[id=6425, name=Thread-2339, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    8) Thread[id=6455, name=zkCallback-1200-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=6451, name=Thread-2341, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)   10) Thread[id=6424, name=OverseerHdfsCoreFailoverThread-96014251173085187-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=6445, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[F65295FADAA2F5CD]-SendThread(127.0.0.1:46374), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   2) Thread[id=6454, name=zkCallback-1200-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   3) Thread[id=6456, name=zkCallback-1200-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   4) Thread[id=6450, name=OverseerHdfsCoreFailoverThread-96014272158892035-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Throwable.fillInStackTrace(java.base@9-ea/Native Method)
        at java.lang.Throwable.fillInStackTrace(java.base@9-ea/Throwable.java:788)
        at java.lang.Throwable.<init>(java.base@9-ea/Throwable.java:271)
        at java.lang.Exception.<init>(java.base@9-ea/Exception.java:66)
        at java.lang.InterruptedException.<init>(java.base@9-ea/InterruptedException.java:67)
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   5) Thread[id=6453, name=zkCallback-1200-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   6) Thread[id=6419, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[F65295FADAA2F5CD]-SendThread(127.0.0.1:41542), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   7) Thread[id=6425, name=Thread-2339, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   8) Thread[id=6455, name=zkCallback-1200-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=6451, name=Thread-2341, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
  10) Thread[id=6424, name=OverseerHdfsCoreFailoverThread-96014251173085187-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
	at __randomizedtesting.SeedInfo.seed([F65295FADAA2F5CD]:0)


FAILED:  org.apache.solr.cloud.AsyncMigrateRouteKeyTest.test

Error Message:
Error from server at http://127.0.0.1:40529/jrdu/r/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /jrdu/r/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:40529/jrdu/r/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /jrdu/r/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([F65295FADAA2F5CD:7E06AA20745E9835]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest(MigrateRouteKeyTest.java:135)
	at org.apache.solr.cloud.AsyncMigrateRouteKeyTest.test(AsyncMigrateRouteKeyTest.java:44)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
Error from server at http://127.0.0.1:41525/yo_/j: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore

Stack Trace:
java.lang.AssertionError: Error from server at http://127.0.0.1:41525/yo_/j: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore
	at __randomizedtesting.SeedInfo.seed([F65295FADAA2F5CD:7E06AA20745E9835]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:162)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:71)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:34595/b_ojc/l/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /b_ojc/l/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:34595/b_ojc/l/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /b_ojc/l/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([F65295FADAA2F5CD:7E06AA20745E9835]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:152)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration

Error Message:
Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: Timed out getting shard id for core: testcore

Stack Trace:
org.apache.solr.common.SolrException: Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: Timed out getting shard id for core: testcore
	at __randomizedtesting.SeedInfo.seed([F65295FADAA2F5CD:48D9F355A3D8FBF8]:0)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:785)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:718)
	at org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration(ClusterStateUpdateTest.java:158)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Unable to create core [testcore]
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:828)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:751)
	... 41 more
Caused by: org.apache.solr.common.SolrException: Timed out getting shard id for core: testcore
	at org.apache.solr.cloud.ZkController.waitForShardId(ZkController.java:1407)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1362)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	... 42 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:38719/_g

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:38719/_g
	at __randomizedtesting.SeedInfo.seed([F65295FADAA2F5CD:7E06AA20745E9835]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:601)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:399)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testNoConfigSetExist(CollectionsAPIDistributedZkTest.java:601)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:176)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(java.base@9-ea/Native Method)
	at java.net.SocketInputStream.socketRead(java.base@9-ea/SocketInputStream.java:116)
	at java.net.SocketInputStream.read(java.base@9-ea/SocketInputStream.java:170)
	at java.net.SocketInputStream.read(java.base@9-ea/SocketInputStream.java:141)
	at sun.security.ssl.SSLSocketInputRecord.read(java.base@9-ea/SSLSocketInputRecord.java:425)
	at sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(java.base@9-ea/SSLSocketInputRecord.java:65)
	at sun.security.ssl.SSLSocketImpl.bytesInCompletePacket(java.base@9-ea/SSLSocketImpl.java:911)
	at sun.security.ssl.AppInputStream.read(java.base@9-ea/AppInputStream.java:144)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:495)
	... 47 more


FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Error from server at http://127.0.0.1:44869/gw/pp/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /gw/pp/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44869/gw/pp/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /gw/pp/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([F65295FADAA2F5CD:7E06AA20745E9835]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.indexDoc(AbstractFullDistribZkTestBase.java:775)
	at org.apache.solr.BaseDistributedSearchTestCase.indexr(BaseDistributedSearchTestCase.java:472)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.addUpdateDelete(FullSolrCloudDistribCmdsTest.java:472)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:81)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.LeaderElectionIntegrationTest.testSimpleSliceLeaderElection

Error Message:
Could not start container:7000. Reason:class org.apache.solr.common.SolrException Error CREATEing SolrCore 'collection1_shard1': Unable to create core [collection1_shard1] Caused by: Timed out getting coreNodeName for collection1_shard1

Stack Trace:
java.lang.AssertionError: Could not start container:7000. Reason:class org.apache.solr.common.SolrException Error CREATEing SolrCore 'collection1_shard1': Unable to create core [collection1_shard1] Caused by: Timed out getting coreNodeName for collection1_shard1
	at __randomizedtesting.SeedInfo.seed([F65295FADAA2F5CD:A86658B499DD692A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.LeaderElectionIntegrationTest.setUp(LeaderElectionIntegrationTest.java:102)
	at jdk.internal.reflect.GeneratedMethodAccessor45.invoke(Unknown Source)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:905)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout

Error Message:
Could not start container:7000. Reason:class org.apache.solr.common.SolrException Error CREATEing SolrCore 'collection1_shard1': Unable to create core [collection1_shard1] Caused by: Timed out getting coreNodeName for collection1_shard1

Stack Trace:
java.lang.AssertionError: Could not start container:7000. Reason:class org.apache.solr.common.SolrException Error CREATEing SolrCore 'collection1_shard1': Unable to create core [collection1_shard1] Caused by: Timed out getting coreNodeName for collection1_shard1
	at __randomizedtesting.SeedInfo.seed([F65295FADAA2F5CD:5F653059FD877D26]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.LeaderElectionIntegrationTest.setUp(LeaderElectionIntegrationTest.java:102)
	at jdk.internal.reflect.GeneratedMethodAccessor45.invoke(Unknown Source)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:905)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:


Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([F65295FADAA2F5CD:7E06AA20745E9835]:0)
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertTrue(Assert.java:54)
	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:127)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.ShardRoutingTest.test

Error Message:
Error from server at http://127.0.0.1:46016/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:46016/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([F65295FADAA2F5CD:7E06AA20745E9835]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClie

[...truncated too long message...]

bove limit:
   [junit4]   2> 
   [junit4]   2> 	7	/solr/clusterprops.json
   [junit4]   2> 	7	/solr/clusterstate.json
   [junit4]   2> 	4	/solr/collections/nodes_used_collection/state.json
   [junit4]   2> 	4	/solr/collections/collection1/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	7	/solr/live_nodes
   [junit4]   2> 	7	/solr/collections
   [junit4]   2> 	5	/solr/overseer/collection-queue-work
   [junit4]   2> 	4	/solr/overseer/queue
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CollectionsAPIDistributedZkTest -Dtests.method=test -Dtests.seed=F65295FADAA2F5CD -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=mn-Cyrl-MN -Dtests.timezone=Asia/Bishkek -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR    370s J1 | CollectionsAPIDistributedZkTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:38719/_g
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F65295FADAA2F5CD:7E06AA20745E9835]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:601)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:399)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testNoConfigSetExist(CollectionsAPIDistributedZkTest.java:601)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:176)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(java.base@9-ea/Native Method)
   [junit4]    > 	at java.net.SocketInputStream.socketRead(java.base@9-ea/SocketInputStream.java:116)
   [junit4]    > 	at java.net.SocketInputStream.read(java.base@9-ea/SocketInputStream.java:170)
   [junit4]    > 	at java.net.SocketInputStream.read(java.base@9-ea/SocketInputStream.java:141)
   [junit4]    > 	at sun.security.ssl.SSLSocketInputRecord.read(java.base@9-ea/SSLSocketInputRecord.java:425)
   [junit4]    > 	at sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(java.base@9-ea/SSLSocketInputRecord.java:65)
   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.bytesInCompletePacket(java.base@9-ea/SSLSocketImpl.java:911)
   [junit4]    > 	at sun.security.ssl.AppInputStream.read(java.base@9-ea/AppInputStream.java:144)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
   [junit4]    > 	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:495)
   [junit4]    > 	... 47 more
   [junit4]   2> 4422429 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[F65295FADAA2F5CD]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_F65295FADAA2F5CD-001
   [junit4]   2> Jun 04, 2016 5:45:59 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 3 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=false,coord=crazy): {}, locale=mn-Cyrl-MN, timezone=Asia/Bishkek
   [junit4]   2> NOTE: Linux 4.4.0-21-generic i386/Oracle Corporation 9-ea (32-bit)/cpus=12,threads=1,free=159324360,total=478150656
   [junit4]   2> NOTE: All tests run in this JVM: [TestLMJelinekMercerSimilarityFactory, TestReRankQParserPlugin, TestReqParamsAPI, TestQueryTypes, TestDistributedGrouping, DistanceFunctionTest, TestManagedSchemaAPI, AddSchemaFieldsUpdateProcessorFactoryTest, TestFieldCacheSortRandom, TestObjectReleaseTracker, SpellingQueryConverterTest, TestQuerySenderNoQuery, ConvertedLegacyTest, TestSolr4Spatial2, TestPivotHelperCode, TestLeaderInitiatedRecoveryThread, SaslZkACLProviderTest, CdcrReplicationHandlerTest, LoggingHandlerTest, SpellCheckCollatorTest, TestUninvertingReader, DistributedQueryComponentCustomSortTest, TestSolrConfigHandlerCloud, RuleEngineTest, SuggesterFSTTest, TestConfigSetsAPIZkFailure, SoftAutoCommitTest, HighlighterTest, TestBM25SimilarityFactory, HdfsThreadLeakTest, SolrPluginUtilsTest, ShardRoutingCustomTest, SpellCheckComponentTest, TestUniqueKeyFieldResource, IndexBasedSpellCheckerTest, HdfsCollectionsAPIDistributedZkTest, SecurityConfHandlerTest, TestSha256AuthenticationProvider, TestExceedMaxTermLength, TestReloadAndDeleteDocs, ClusterStateUpdateTest, TestSearcherReuse, TestInfoStreamLogging, HdfsWriteToMultipleCollectionsTest, TestLegacyFieldCache, TestCloudDeleteByQuery, ConfigSetsAPITest, TestReplicaProperties, CollectionsAPIAsyncDistributedZkTest, ZkSolrClientTest, SpatialFilterTest, TestFieldCacheReopen, TestDFISimilarityFactory, NotRequiredUniqueKeyTest, SharedFSAutoReplicaFailoverUtilsTest, MultiTermTest, TestTrie, HighlighterMaxOffsetTest, TestLFUCache, SuggestComponentTest, TestCollectionAPI, TestCodecSupport, DistributedIntervalFacetingTest, MinimalSchemaTest, TestMissingGroups, HttpPartitionTest, TestSolrCloudWithKerberosAlt, TestUseDocValuesAsStored2, CdcrUpdateLogTest, NumericFieldsTest, ShardRoutingTest, BasicFunctionalityTest, AsyncMigrateRouteKeyTest, TestFieldTypeCollectionResource, TestSolrDeletionPolicy2, DocumentBuilderTest, BlockDirectoryTest, SolrXmlInZkTest, DistributedFacetPivotSmallAdvancedTest, TestSubQueryTransformerDistrib, WordBreakSolrSpellCheckerTest, CollectionsAPIDistributedZkTest]
   [junit4] Completed [581/613 (12!)] on J1 in 370.68s, 1 test, 1 error <<< FAILURES!

[...truncated 149 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 2 suite-level errors, 9 errors, 4 failures, 1325 ignored (58 assumptions) [seed: F65295FADAA2F5CD]

Total time: 125 minutes 10 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-6.x-Linux (32bit/jdk1.8.0_92) - Build # 829 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/829/
Java: 32bit/jdk1.8.0_92 -server -XX:+UseParallelGC

14 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest:     1) Thread[id=5125, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[CBF865CB00F24CAD]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    2) Thread[id=5155, name=Thread-1863, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    3) Thread[id=5160, name=zkCallback-866-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=5128, name=OverseerCollectionConfigSetProcessor-96013014568140803-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=5159, name=zkCallback-866-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=5124, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[CBF865CB00F24CAD]-SendThread(127.0.0.1:33617), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    7) Thread[id=5130, name=Thread-1861, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    8) Thread[id=5150, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[CBF865CB00F24CAD]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    9) Thread[id=5153, name=OverseerCollectionConfigSetProcessor-96013035885953027-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=5129, name=OverseerHdfsCoreFailoverThread-96013014568140803-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)   11) Thread[id=5152, name=OverseerStateUpdate-96013035885953027-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)         at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)         at java.lang.Thread.run(Thread.java:745)   12) Thread[id=5154, name=OverseerHdfsCoreFailoverThread-96013035885953027-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)   13) Thread[id=5127, name=OverseerStateUpdate-96013014568140803-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)         at java.lang.Thread.run(Thread.java:745)   14) Thread[id=5157, name=zkCallback-866-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   15) Thread[id=5149, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[CBF865CB00F24CAD]-SendThread(127.0.0.1:44022), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   16) Thread[id=5158, name=zkCallback-866-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest: 
   1) Thread[id=5125, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[CBF865CB00F24CAD]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   2) Thread[id=5155, name=Thread-1863, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   3) Thread[id=5160, name=zkCallback-866-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=5128, name=OverseerCollectionConfigSetProcessor-96013014568140803-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=5159, name=zkCallback-866-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=5124, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[CBF865CB00F24CAD]-SendThread(127.0.0.1:33617), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   7) Thread[id=5130, name=Thread-1861, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   8) Thread[id=5150, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[CBF865CB00F24CAD]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   9) Thread[id=5153, name=OverseerCollectionConfigSetProcessor-96013035885953027-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=5129, name=OverseerHdfsCoreFailoverThread-96013014568140803-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
  11) Thread[id=5152, name=OverseerStateUpdate-96013035885953027-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
        at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)
        at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)
        at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)
        at java.lang.Thread.run(Thread.java:745)
  12) Thread[id=5154, name=OverseerHdfsCoreFailoverThread-96013035885953027-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
  13) Thread[id=5127, name=OverseerStateUpdate-96013014568140803-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
        at java.lang.Thread.run(Thread.java:745)
  14) Thread[id=5157, name=zkCallback-866-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  15) Thread[id=5149, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[CBF865CB00F24CAD]-SendThread(127.0.0.1:44022), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  16) Thread[id=5158, name=zkCallback-866-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([CBF865CB00F24CAD]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=5129, name=OverseerHdfsCoreFailoverThread-96013014568140803-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=5154, name=OverseerHdfsCoreFailoverThread-96013035885953027-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=5155, name=Thread-1863, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    4) Thread[id=5160, name=zkCallback-866-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=5159, name=zkCallback-866-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=5157, name=zkCallback-866-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=5149, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[CBF865CB00F24CAD]-SendThread(127.0.0.1:44022), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    8) Thread[id=5158, name=zkCallback-866-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=5124, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[CBF865CB00F24CAD]-SendThread(127.0.0.1:33617), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)   10) Thread[id=5130, name=Thread-1861, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=5129, name=OverseerHdfsCoreFailoverThread-96013014568140803-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=5154, name=OverseerHdfsCoreFailoverThread-96013035885953027-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=5155, name=Thread-1863, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   4) Thread[id=5160, name=zkCallback-866-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=5159, name=zkCallback-866-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=5157, name=zkCallback-866-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=5149, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[CBF865CB00F24CAD]-SendThread(127.0.0.1:44022), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   8) Thread[id=5158, name=zkCallback-866-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=5124, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[CBF865CB00F24CAD]-SendThread(127.0.0.1:33617), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
  10) Thread[id=5130, name=Thread-1861, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
	at __randomizedtesting.SeedInfo.seed([CBF865CB00F24CAD]:0)


FAILED:  org.apache.solr.cloud.AliasIntegrationTest.test

Error Message:
Error from server at https://127.0.0.1:33242/rwz/b/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /rwz/b/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:33242/rwz/b/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /rwz/b/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([CBF865CB00F24CAD:43AC5A11AE0E2155]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.AliasIntegrationTest.test(AliasIntegrationTest.java:69)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.AsyncMigrateRouteKeyTest.test

Error Message:
Error from server at http://127.0.0.1:38109/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:38109/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([CBF865CB00F24CAD:43AC5A11AE0E2155]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest(MigrateRouteKeyTest.java:135)
	at org.apache.solr.cloud.AsyncMigrateRouteKeyTest.test(AsyncMigrateRouteKeyTest.java:44)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
Error from server at http://127.0.0.1:44908: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore

Stack Trace:
java.lang.AssertionError: Error from server at http://127.0.0.1:44908: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore
	at __randomizedtesting.SeedInfo.seed([CBF865CB00F24CAD:43AC5A11AE0E2155]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:162)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:71)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:44498/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44498/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([CBF865CB00F24CAD:43AC5A11AE0E2155]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:152)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.BasicZkTest.testBasic

Error Message:
No registered leader was found after waiting for 30000ms , collection: collection1 slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 30000ms , collection: collection1 slice: shard1
	at __randomizedtesting.SeedInfo.seed([CBF865CB00F24CAD:600278DEDF2ECA83]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:718)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:685)
	at org.apache.solr.cloud.BasicZkTest.testBasic(BasicZkTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
Error from server at https://127.0.0.1:32861/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:32861/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([CBF865CB00F24CAD:43AC5A11AE0E2155]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(H

[...truncated too long message...]

r@ee5f04{/,null,UNAVAILABLE}
   [junit4]   2> 6360374 INFO  (TEST-UnloadDistributedZkTest.test-seed#[CBF865CB00F24CAD]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40980 40980
   [junit4]   2> 6360395 INFO  (Thread-2303) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:40980 40980
   [junit4]   2> 6360396 WARN  (Thread-2303) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/aliases.json
   [junit4]   2> 	6	/solr/clusterprops.json
   [junit4]   2> 	5	/solr/security.json
   [junit4]   2> 	4	/solr/collections/collection1/state.json
   [junit4]   2> 	4	/solr/configs/conf1
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/live_nodes
   [junit4]   2> 	6	/solr/collections
   [junit4]   2> 	5	/solr/overseer/collection-queue-work
   [junit4]   2> 	4	/solr/overseer/queue
   [junit4]   2> 	3	/solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=UnloadDistributedZkTest -Dtests.method=test -Dtests.seed=CBF865CB00F24CAD -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=uk -Dtests.timezone=Europe/Bratislava -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    682s J2 | UnloadDistributedZkTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:36374
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([CBF865CB00F24CAD:43AC5A11AE0E2155]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:601)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]    > 	at org.apache.solr.cloud.UnloadDistributedZkTest.testCoreUnloadAndLeaders(UnloadDistributedZkTest.java:186)
   [junit4]    > 	at org.apache.solr.cloud.UnloadDistributedZkTest.test(UnloadDistributedZkTest.java:67)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:170)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:141)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
   [junit4]    > 	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:495)
   [junit4]    > 	... 46 more
   [junit4]   2> 6360398 INFO  (SUITE-UnloadDistributedZkTest-seed#[CBF865CB00F24CAD]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.UnloadDistributedZkTest_CBF865CB00F24CAD-001
   [junit4]   2> Jun 04, 2016 12:33:33 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene60): {}, docValues:{}, maxPointsInLeafNode=2002, maxMBSortInHeap=7.242712670901151, sim=RandomSimilarity(queryNorm=true,coord=crazy): {}, locale=uk, timezone=Europe/Bratislava
   [junit4]   2> NOTE: Linux 4.4.0-21-generic i386/Oracle Corporation 1.8.0_92 (32-bit)/cpus=12,threads=1,free=119918008,total=323223552
   [junit4]   2> NOTE: All tests run in this JVM: [SpatialHeatmapFacetsTest, BadIndexSchemaTest, TestCollationFieldDocValues, HLLUtilTest, QueryParsingTest, TestPerFieldSimilarity, SliceStateTest, TestNoOpRegenerator, TestPostingsSolrHighlighter, DateMathParserTest, TestSolrQueryParser, TestCharFilters, ExplicitHLLTest, TestLockTree, TestRecovery, SimpleCollectionCreateDeleteTest, BinaryUpdateRequestHandlerTest, TestTrackingShardHandlerFactory, TestDistributedStatsComponentCardinality, BJQParserTest, TestMacros, BufferStoreTest, AutoCommitTest, TestSolrDeletionPolicy1, TestRTimerTree, ShowFileRequestHandlerTest, TestTolerantSearch, TestFastWriter, TestReversedWildcardFilterFactory, DirectUpdateHandlerTest, TestDistributedMissingSort, TestUtils, CachingDirectoryFactoryTest, TestRestoreCore, TestInitQParser, CleanupOldIndexTest, LukeRequestHandlerTest, HardAutoCommitTest, TestExtendedDismaxParser, TestCryptoKeys, DistributedQueryComponentOptimizationTest, ScriptEngineTest, TestSuggestSpellingConverter, StressHdfsTest, TestPKIAuthenticationPlugin, TestStressRecovery, TestFieldCacheVsDocValues, TestTolerantUpdateProcessorRandomCloud, NoCacheHeaderTest, TestFuzzyAnalyzedSuggestions, TestAddFieldRealTimeGet, SuggesterWFSTTest, TestZkChroot, PluginInfoTest, HdfsRestartWhileUpdatingTest, BasicAuthIntegrationTest, DataDrivenBlockJoinTest, TestExactSharedStatsCache, TestSlowCompositeReaderWrapper, TestHdfsUpdateLog, TestSolrConfigHandlerConcurrent, HdfsDirectoryFactoryTest, AddBlockUpdateTest, SuggesterTest, DirectSolrConnectionTest, TestComplexPhraseQParserPlugin, TestXIncludeConfig, SimpleFacetsTest, TestFilteredDocIdSet, TestSolrQueryParserDefaultOperatorResource, TestDefaultSearchFieldResource, TestWriterPerf, FieldAnalysisRequestHandlerTest, DocExpirationUpdateProcessorFactoryTest, TestShardHandlerFactory, FieldMutatingUpdateProcessorTest, TestNonDefinedSimilarityFactory, TestManagedStopFilterFactory, DeleteInactiveReplicaTest, TestJmxIntegration, DistributedFacetPivotSmallTest, EnumFieldTest, CdcrRequestHandlerTest, CurrencyFieldXmlFileTest, DistributedVersionInfoTest, TestCSVResponseWriter, TestInitParams, TestSchemaSimilarityResource, PreAnalyzedUpdateProcessorTest, TestSolrJ, TestReload, ClusterStateTest, TestSolrIndexConfig, TestSearchPerf, TestIndexingPerformance, TestRequestForwarding, SimpleMLTQParserTest, TestExclusionRuleCollectionAccess, CollectionStateFormat2Test, BasicDistributedZk2Test, UnloadDistributedZkTest]
   [junit4] Completed [613/613 (11!)] on J2 in 682.86s, 1 test, 1 error <<< FAILURES!

[...truncated 20 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 2 suite-level errors, 9 errors, 3 failures, 1386 ignored (65 assumptions) [seed: CBF865CB00F24CAD]

Total time: 130 minutes 14 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS-EA] Lucene-Solr-6.x-Linux (64bit/jdk-9-ea+121) - Build # 828 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/828/
Java: 64bit/jdk-9-ea+121 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
FAILED:  org.apache.solr.cloud.overseer.ZkStateReaderTest.testStateFormatUpdateWithTimeDelay

Error Message:


Stack Trace:
java.util.concurrent.TimeoutException
	at __randomizedtesting.SeedInfo.seed([8B59EC9415139483:F4C75B117C71B909]:0)
	at org.apache.solr.common.cloud.ZkStateReader.waitForState(ZkStateReader.java:1200)
	at org.apache.solr.cloud.overseer.ZkStateReaderTest.testStateFormatUpdate(ZkStateReaderTest.java:126)
	at org.apache.solr.cloud.overseer.ZkStateReaderTest.testStateFormatUpdateWithTimeDelay(ZkStateReaderTest.java:54)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)




Build Log:
[...truncated 12105 lines...]
   [junit4] Suite: org.apache.solr.cloud.overseer.ZkStateReaderTest
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.overseer.ZkStateReaderTest_8B59EC9415139483-001/init-core-data-001
   [junit4]   2> 1496186 INFO  (SUITE-ZkStateReaderTest-seed#[8B59EC9415139483]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1496191 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testStateFormatUpdateWithExplicitRefresh
   [junit4]   2> 1496191 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1496191 INFO  (Thread-4193) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1496191 INFO  (Thread-4193) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1496291 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.ZkTestServer start zk server on port:46719
   [junit4]   2> 1496291 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1496291 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1496292 INFO  (zkCallback-20798-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@68acd97f name:ZooKeeperConnection Watcher:127.0.0.1:46719 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1496292 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1496293 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1496293 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1496293 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1496293 INFO  (zkCallback-20799-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@28298974 name:ZooKeeperConnection Watcher:127.0.0.1:46719 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1496293 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1496294 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1496294 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 1496307 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1496307 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1496308 INFO  (zkCallback-20800-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@177df463 name:ZooKeeperConnection Watcher:127.0.0.1:46719/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1496308 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1496308 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1496308 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 1496309 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 1496309 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 1496309 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 1496310 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 1496310 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 1496310 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 1496311 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 1496311 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Deleting data for [c1]
   [junit4]   2> 1496311 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 1496318 INFO  (zkCallback-20800-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 1496318 INFO  (zkCallback-20800-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 1496319 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 1496319 INFO  (zkCallback-20800-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/c1/state.json] for collection [c1] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 1496319 INFO  (zkCallback-20800-thread-1) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 1496319 INFO  (zkCallback-20800-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 1496320 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46719 46719
   [junit4]   2> 1501333 INFO  (Thread-4193) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46719 46719
   [junit4]   2> 1501333 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8B59EC9415139483]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStateFormatUpdateWithExplicitRefresh
   [junit4]   2> 1501335 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testStateFormatUpdateWithExplicitRefreshLazy
   [junit4]   2> 1501335 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1501335 INFO  (Thread-4194) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1501335 INFO  (Thread-4194) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1501435 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.ZkTestServer start zk server on port:36513
   [junit4]   2> 1501435 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1501435 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1501436 INFO  (zkCallback-20802-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4c2833a4 name:ZooKeeperConnection Watcher:127.0.0.1:36513 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1501436 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1501436 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1501437 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1501437 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1501437 INFO  (zkCallback-20803-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@428be70 name:ZooKeeperConnection Watcher:127.0.0.1:36513 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1501437 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1501437 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1501437 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 1501439 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1501440 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1501440 INFO  (zkCallback-20804-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@20ccca8 name:ZooKeeperConnection Watcher:127.0.0.1:36513/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1501440 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1501440 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1501440 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 1501441 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 1501441 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 1501441 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 1501442 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 1501442 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 1501442 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 1501442 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 1501443 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 1501443 INFO  (zkCallback-20804-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 1501443 INFO  (zkCallback-20804-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 1501444 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 1501444 INFO  (zkCallback-20804-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 1501444 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:36513 36513
   [junit4]   2> 1501574 INFO  (Thread-4194) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:36513 36513
   [junit4]   2> 1501575 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8B59EC9415139483]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStateFormatUpdateWithExplicitRefreshLazy
   [junit4]   2> 1501576 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testWatchedCollectionCreation
   [junit4]   2> 1501576 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1501576 INFO  (Thread-4195) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1501576 INFO  (Thread-4195) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1501676 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.ZkTestServer start zk server on port:38663
   [junit4]   2> 1501677 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1501677 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1501678 INFO  (zkCallback-20806-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6ba78b85 name:ZooKeeperConnection Watcher:127.0.0.1:38663 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1501678 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1501679 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1501679 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1501679 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1501679 INFO  (zkCallback-20807-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3a49d01a name:ZooKeeperConnection Watcher:127.0.0.1:38663 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1501679 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1501680 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1501680 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 1501680 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1501681 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1501681 INFO  (zkCallback-20808-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@48a5e9fa name:ZooKeeperConnection Watcher:127.0.0.1:38663/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1501681 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1501681 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1501681 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 1501682 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 1501682 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 1501683 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 1501683 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 1501683 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 1501684 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 1501684 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 1501684 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Deleting data for [c1]
   [junit4]   2> 1501685 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 1501685 INFO  (zkCallback-20808-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 1501685 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 1501686 INFO  (zkCallback-20808-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/c1/state.json] for collection [c1] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 1501686 INFO  (zkCallback-20808-thread-1) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 1501687 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38663 38663
   [junit4]   2> 1501719 INFO  (Thread-4195) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38663 38663
   [junit4]   2> 1501719 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8B59EC9415139483]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testWatchedCollectionCreation
   [junit4]   2> 1501721 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testStateFormatUpdateWithTimeDelayLazy
   [junit4]   2> 1501721 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1501721 INFO  (Thread-4196) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1501721 INFO  (Thread-4196) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1501821 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.ZkTestServer start zk server on port:36487
   [junit4]   2> 1501821 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1501822 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1501823 INFO  (zkCallback-20810-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6878f24b name:ZooKeeperConnection Watcher:127.0.0.1:36487 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1501823 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1501824 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1501824 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1501824 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1501824 INFO  (zkCallback-20811-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@14af2bf0 name:ZooKeeperConnection Watcher:127.0.0.1:36487 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1501824 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1501825 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1501825 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 1501825 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1501826 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1501826 INFO  (zkCallback-20812-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5c2f157f name:ZooKeeperConnection Watcher:127.0.0.1:36487/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1501826 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1501826 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1501826 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 1501827 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 1501827 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 1501827 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 1501828 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 1501828 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 1501828 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 1501828 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 1501828 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 1501829 INFO  (zkCallback-20812-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 1501829 INFO  (zkCallback-20812-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 1501829 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Deleting data for [c1]
   [junit4]   2> 1501830 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 1501830 INFO  (zkCallback-20812-thread-1) [    ] o.a.s.c.c.ZkStateReader Uninteresting collection c1
   [junit4]   2> 1501830 INFO  (zkCallback-20812-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 1501830 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 1501831 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:36487 36487
   [junit4]   2> 1501870 INFO  (Thread-4196) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:36487 36487
   [junit4]   2> 1501871 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8B59EC9415139483]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStateFormatUpdateWithTimeDelayLazy
   [junit4]   2> 1501872 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testStateFormatUpdateWithTimeDelay
   [junit4]   2> 1501873 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1501873 INFO  (Thread-4197) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1501873 INFO  (Thread-4197) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1501973 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.ZkTestServer start zk server on port:41587
   [junit4]   2> 1501973 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1501974 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1501974 INFO  (zkCallback-20814-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@52ac30eb name:ZooKeeperConnection Watcher:127.0.0.1:41587 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1501974 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1501975 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1501975 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1501975 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1501976 INFO  (zkCallback-20815-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4a44055d name:ZooKeeperConnection Watcher:127.0.0.1:41587 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1501976 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1501976 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1501976 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 1501977 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1501977 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1501977 INFO  (zkCallback-20816-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@30a68e58 name:ZooKeeperConnection Watcher:127.0.0.1:41587/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1501978 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1501978 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1501978 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 1501979 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 1501979 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 1501980 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 1501980 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 1501980 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 1501980 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 1501981 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 1501981 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Deleting data for [c1]
   [junit4]   2> 1501981 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 1501981 INFO  (zkCallback-20816-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 1501982 INFO  (zkCallback-20816-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 1501982 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 1501982 INFO  (zkCallback-20816-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/c1/state.json] for collection [c1] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 1501982 INFO  (zkCallback-20816-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 1501982 INFO  (zkCallback-20816-thread-1) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 1531983 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:41587 41587
   [junit4]   2> 1532083 INFO  (Thread-4197) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:41587 41587
   [junit4]   2> 1532084 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8B59EC9415139483]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStateFormatUpdateWithTimeDelay
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ZkStateReaderTest -Dtests.method=testStateFormatUpdateWithTimeDelay -Dtests.seed=8B59EC9415139483 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=smn -Dtests.timezone=Antarctica/DumontDUrville -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   30.2s J2 | ZkStateReaderTest.testStateFormatUpdateWithTimeDelay <<<
   [junit4]    > Throwable #1: java.util.concurrent.TimeoutException
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8B59EC9415139483:F4C75B117C71B909]:0)
   [junit4]    > 	at org.apache.solr.common.cloud.ZkStateReader.waitForState(ZkStateReader.java:1200)
   [junit4]    > 	at org.apache.solr.cloud.overseer.ZkStateReaderTest.testStateFormatUpdate(ZkStateReaderTest.java:126)
   [junit4]    > 	at org.apache.solr.cloud.overseer.ZkStateReaderTest.testStateFormatUpdateWithTimeDelay(ZkStateReaderTest.java:54)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]   2> 1532086 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testExternalCollectionWatchedNotWatched
   [junit4]   2> 1532086 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1532087 INFO  (Thread-4198) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1532087 INFO  (Thread-4198) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1532187 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.ZkTestServer start zk server on port:39850
   [junit4]   2> 1532187 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1532187 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1532188 INFO  (zkCallback-20818-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@a60f35c name:ZooKeeperConnection Watcher:127.0.0.1:39850 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1532188 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1532188 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1532189 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1532189 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1532189 INFO  (zkCallback-20819-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3ab38fc0 name:ZooKeeperConnection Watcher:127.0.0.1:39850 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1532189 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1532190 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1532190 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 1532191 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1532191 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1532191 INFO  (zkCallback-20820-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@23990d04 name:ZooKeeperConnection Watcher:127.0.0.1:39850/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1532191 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1532191 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1532191 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 1532192 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 1532193 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 1532193 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 1532194 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 1532194 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 1532195 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 1532195 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 1532196 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 1532196 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 1532197 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 1532197 INFO  (zkCallback-20820-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 1532202 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39850 39850
   [junit4]   2> 1532261 INFO  (Thread-4198) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:39850 39850
   [junit4]   2> 1532261 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8B59EC9415139483]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testExternalCollectionWatchedNotWatched
   [junit4]   2> 1532262 INFO  (SUITE-ZkStateReaderTest-seed#[8B59EC9415139483]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.overseer.ZkStateReaderTest_8B59EC9415139483-001
   [junit4]   2> Jun 04, 2016 9:25:01 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 5 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene60, sim=ClassicSimilarity, locale=smn, timezone=Antarctica/DumontDUrville
   [junit4]   2> NOTE: Linux 4.4.0-21-generic amd64/Oracle Corporation 9-ea (64-bit)/cpus=12,threads=1,free=225657864,total=515964928
   [junit4]   2> NOTE: All tests run in this JVM: [TestSubQueryTransformer, CoreAdminHandlerTest, RemoteQueryErrorTest, ScriptEngineTest, DistributedQueryComponentOptimizationTest, FieldMutatingUpdateProcessorTest, TestFieldTypeResource, TestMacros, TestManagedResource, TestRTimerTree, TestStressCloudBlindAtomicUpdates, SpatialHeatmapFacetsTest, VMParamsZkACLAndCredentialsProvidersTest, RegexBoostProcessorTest, CollectionStateFormat2Test, TestMinMaxOnMultiValuedField, TestStressRecovery, UpdateParamsTest, TestAuthenticationFramework, TestTrackingShardHandlerFactory, DocExpirationUpdateProcessorFactoryTest, SimpleCollectionCreateDeleteTest, TestPHPSerializedResponseWriter, TestCoreDiscovery, NoCacheHeaderTest, BlockJoinFacetSimpleTest, TestFuzzyAnalyzedSuggestions, TestExactSharedStatsCache, SampleTest, TestNonDefinedSimilarityFactory, OverseerTest, TestRequestForwarding, CachingDirectoryFactoryTest, OverriddenZkACLAndCredentialsProvidersTest, TestJmxIntegration, HdfsNNFailoverTest, TestRandomCollapseQParserPlugin, SolrTestCaseJ4Test, SearchHandlerTest, BigEndianAscendingWordDeserializerTest, TestDynamicLoading, CircularListTest, TestSolrQueryResponse, TestCloudInspectUtil, AtomicUpdatesTest, TestSweetSpotSimilarityFactory, TestSolrJ, MigrateRouteKeyTest, AddBlockUpdateTest, HighlighterConfigTest, TestSolrConfigHandlerConcurrent, TestManagedStopFilterFactory, TestNoOpRegenerator, TestShardHandlerFactory, TestJmxMonitoredMap, TestSolrDeletionPolicy1, TestCharFilters, TestShortCircuitedRequests, AssignTest, CoreMergeIndexesAdminHandlerTest, TestFieldCacheSanityChecker, TestSolrXml, ConnectionManagerTest, QueryParsingTest, ShowFileRequestHandlerTest, StatsComponentTest, ExplicitHLLTest, TestUtils, TestAddFieldRealTimeGet, DirectSolrConnectionTest, FullSolrCloudDistribCmdsTest, CleanupOldIndexTest, AutoCommitTest, DocValuesMultiTest, DistribDocExpirationUpdateProcessorTest, LeaderElectionTest, ClusterStateTest, RecoveryZkTest, TestInitQParser, TestBulkSchemaAPI, TestWriterPerf, TestElisionMultitermQuery, DeleteShardTest, TestReload, TestSuggestSpellingConverter, SegmentsInfoRequestHandlerTest, TestInitParams, BasicDistributedZk2Test, HardAutoCommitTest, TestLockTree, DataDrivenBlockJoinTest, OverseerStatusTest, TestCollationField, SliceStateTest, TestGraphMLResponseWriter, TestRandomRequestDistribution, AnalyticsMergeStrategyTest, TestRandomFaceting, ZkSolrClientTest, TestDistributedSearch, TestLazyCores, CurrencyFieldOpenExchangeTest, CurrencyFieldXmlFileTest, AnalysisAfterCoreReloadTest, TestExtendedDismaxParser, SpellCheckCollatorTest, SuggesterTSTTest, SuggesterTest, TestStressLucene, SuggesterWFSTTest, WordBreakSolrSpellCheckerTest, SchemaVersionSpecificBehaviorTest, SolrCoreCheckLockOnStartupTest, TestPseudoReturnFields, TestUpdate, TestWordDelimiterFilterFactory, DirectUpdateHandlerOptimizeTest, StatelessScriptUpdateProcessorFactoryTest, DebugComponentTest, LukeRequestHandlerTest, TestReversedWildcardFilterFactory, PrimitiveFieldTypeTest, XmlUpdateRequestHandlerTest, TestValueSourceCache, TermVectorComponentTest, RequiredFieldsTest, TestSolrQueryParser, FastVectorHighlighterTest, IndexSchemaRuntimeFieldTest, ReturnFieldsTest, TestCSVResponseWriter, BinaryUpdateRequestHandlerTest, AlternateDirectoryTest, TestSolrIndexConfig, TestMergePolicyConfig, TestBinaryField, TestSolrCoreProperties, TestIBSimilarityFactory, ResourceLoaderTest, PluginInfoTest, SpellingQueryConverterTest, ZkNodePropsTest, TestRTGBase, DistributedIntervalFacetingTest, CollectionsAPISolrJTest, LeaderFailoverAfterPartitionTest, ShardSplitTest, TestSolrCloudWithKerberosAlt, TestStressLiveNodes, HdfsBasicDistributedZk2Test, HdfsBasicDistributedZkTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsRecoverLeaseTest, HdfsRecoveryZkTest, HdfsRestartWhileUpdatingTest, HdfsThreadLeakTest, HdfsTlogReplayBufferedWhileIndexingTest, HdfsWriteToMultipleCollectionsTest, StressHdfsTest, TestClusterStateMutator, ZkStateReaderTest]
   [junit4] Completed [484/613 (1!)] on J2 in 36.59s, 6 tests, 1 error <<< FAILURES!

[...truncated 411 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 1 error, 118 ignored (72 assumptions) [seed: 8B59EC9415139483]

Total time: 51 minutes 34 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-6.x-Linux (32bit/jdk1.8.0_92) - Build # 827 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/827/
Java: 32bit/jdk1.8.0_92 -client -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.cloud.TestRequestStatusCollectionAPI.test

Error Message:
expected:<found [1001] in [completed] tasks> but was:<found [1001] in [running] tasks>

Stack Trace:
org.junit.ComparisonFailure: expected:<found [1001] in [completed] tasks> but was:<found [1001] in [running] tasks>
	at __randomizedtesting.SeedInfo.seed([C4F558B7B5850C5E:4CA1676D1B7961A6]:0)
	at org.junit.Assert.assertEquals(Assert.java:125)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.cloud.TestRequestStatusCollectionAPI.test(TestRequestStatusCollectionAPI.java:111)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10976 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestRequestStatusCollectionAPI
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/init-core-data-001
   [junit4]   2> 548516 INFO  (SUITE-TestRequestStatusCollectionAPI-seed#[C4F558B7B5850C5E]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 548517 INFO  (SUITE-TestRequestStatusCollectionAPI-seed#[C4F558B7B5850C5E]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 548518 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 548518 INFO  (Thread-1774) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 548518 INFO  (Thread-1774) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 548618 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.ZkTestServer start zk server on port:46768
   [junit4]   2> 548619 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 548619 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 548622 INFO  (zkCallback-23989-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@15ec055 name:ZooKeeperConnection Watcher:127.0.0.1:46768 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 548622 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 548622 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 548622 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 548624 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 548624 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 548625 INFO  (zkCallback-23990-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1940d11 name:ZooKeeperConnection Watcher:127.0.0.1:46768/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 548625 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 548625 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 548625 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 548627 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards
   [junit4]   2> 548628 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection
   [junit4]   2> 548628 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards
   [junit4]   2> 548629 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 548629 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 548631 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 548631 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
   [junit4]   2> 548632 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 548632 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 548633 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 548634 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 548635 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 548635 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt
   [junit4]   2> 548636 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 548636 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml
   [junit4]   2> 548637 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 548637 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 548638 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 548638 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 548638 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 548639 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 548639 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 548640 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 548640 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 548640 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 548713 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001/cores/collection1
   [junit4]   2> 548715 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 548716 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@11ae628{/,null,AVAILABLE}
   [junit4]   2> 548716 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.e.j.s.ServerConnector Started ServerConnector@31b338{HTTP/1.1,[http/1.1]}{127.0.0.1:34306}
   [junit4]   2> 548716 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.e.j.s.Server Started @550264ms
   [junit4]   2> 548716 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/tempDir-001/control/data, hostContext=/, hostPort=34306, coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001/cores}
   [junit4]   2> 548717 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@1d16e93
   [junit4]   2> 548717 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001'
   [junit4]   2> 548717 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 548717 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 548727 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 548728 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 548729 INFO  (zkCallback-23991-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1caf401 name:ZooKeeperConnection Watcher:127.0.0.1:46768/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 548729 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 548730 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 548730 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 548730 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001/solr.xml
   [junit4]   2> 548735 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001/cores
   [junit4]   2> 548735 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.CoreContainer New CoreContainer 24911222
   [junit4]   2> 548735 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001]
   [junit4]   2> 548735 WARN  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.CoreContainer Couldn't add files from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001/lib to classpath: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001/lib
   [junit4]   2> 548736 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 548737 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 548737 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 548737 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 548737 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46768/solr
   [junit4]   2> 548737 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 548737 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 548738 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 548739 INFO  (zkCallback-23994-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@44c55e name:ZooKeeperConnection Watcher:127.0.0.1:46768 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 548739 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 548739 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 548740 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$69/14195583@18438a2
   [junit4]   2> 548740 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 548741 INFO  (zkCallback-23995-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@a178c4 name:ZooKeeperConnection Watcher:127.0.0.1:46768/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 548741 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 548742 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 548743 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 548744 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 548745 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 548746 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 548747 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 548747 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 548748 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 548748 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 548749 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 548749 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 548749 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 548750 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 548751 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 548751 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 548752 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96011797137522692-127.0.0.1:34306_-n_0000000000
   [junit4]   2> 548752 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:34306_
   [junit4]   2> 548752 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 548752 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.Overseer Overseer (id=96011797137522692-127.0.0.1:34306_-n_0000000000) starting
   [junit4]   2> 548753 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 548756 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 548757 INFO  (OverseerCollectionConfigSetProcessor-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 548757 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 548757 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34306_
   [junit4]   2> 548758 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:34306_
   [junit4]   2> 548758 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 548759 INFO  (zkCallback-23995-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 548759 INFO  (zkCallback-23995-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 548805 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 548805 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 548805 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001/cores
   [junit4]   2> 548806 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=control_collection, coreNodeName=, dataDir=data/, shard=}
   [junit4]   2> 548806 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.CorePropertiesLocator Found core collection1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001/cores/collection1
   [junit4]   2> 548806 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions
   [junit4]   2> 548807 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.s.SolrDispatchFilter user.dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 548807 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 548807 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
   [junit4]   2> 548807 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:34306_    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 548807 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.ZkController Collection control_collection not visible yet, but flagging it so a watch is registered when it becomes visible
   [junit4]   2> 548808 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.c.ZkStateReader Deleting data for [control_collection]
   [junit4]   2> 548808 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:34306",
   [junit4]   2>   "node_name":"127.0.0.1:34306_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 548808 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:34306",
   [junit4]   2>   "node_name":"127.0.0.1:34306_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 548808 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.o.ClusterStateMutator building a new cName: control_collection
   [junit4]   2> 548808 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 548809 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.ZkController Waiting for coreNodeName for core collection1 in collection control_collection to be assigned
   [junit4]   2> 548809 INFO  (zkCallback-23995-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 548810 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection1
   [junit4]   2> 548810 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.ZkController Check for collection zkNode:control_collection
   [junit4]   2> 548810 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.ZkController Collection zkNode exists
   [junit4]   2> 548810 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/control_collection]
   [junit4]   2> 548810 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.c.ZkStateReader path=[/collections/control_collection] [configName]=[conf1] specified config exists in ZooKeeper
   [junit4]   2> 548811 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001/cores/collection1'
   [junit4]   2> 548811 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 548811 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 548812 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.Config loaded config solrconfig.xml with version 0 
   [junit4]   2> 548816 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 548820 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.1.0
   [junit4]   2> 548828 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 548834 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 548911 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.s.IndexSchema default search field in schema is text
   [junit4]   2> 548912 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 548912 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 548914 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 548930 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 548930 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 548931 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 548931 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001/cores/collection1], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001/cores/collection1/data/]
   [junit4]   2> 548931 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@198a54e
   [junit4]   2> 548931 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001/cores/collection1/data
   [junit4]   2> 548931 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001/cores/collection1/data/index/
   [junit4]   2> 548931 WARN  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Solr index directory '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 548932 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001/cores/collection1/data/index
   [junit4]   2> 548932 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=49, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=8.2119140625, floorSegmentMB=0.6923828125, forceMergeDeletesPctAllowed=23.80956753811972, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 548932 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@f75ed0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@19ad4c0),segFN=segments_1,generation=1}
   [junit4]   2> 548932 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 548933 INFO  (OldIndexDirectoryCleanupThreadForCore-collection1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore Looking for old index directories to cleanup for core collection1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001/cores/collection1/data/
   [junit4]   2> 548933 WARN  (OldIndexDirectoryCleanupThreadForCore-collection1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.DirectoryFactory /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/control-001/cores/collection1/data/ does not point to a valid data directory; skipping clean-up of old index directories.
   [junit4]   2> 548935 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 548935 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 548936 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 548936 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 548936 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 548936 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 548936 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 548936 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 548936 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 548936 WARN  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,args = {defaults={a=A,b=B}}}
   [junit4]   2> 548937 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 548938 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 548939 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 548940 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 548945 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,/update/json/docs,/admin/luke,/export,/get,/admin/properties,/update/json,/admin/threads,/dump,/update/csv,/sql,/graph,/admin/segments,/admin/system,/replication,/config,/stream,/schema,/admin/plugins,/admin/logging,/admin/ping,/update,/admin/file
   [junit4]   2> 548945 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 548945 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 548945 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 548946 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 548946 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 548947 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=42, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 548947 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@f75ed0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@19ad4c0),segFN=segments_1,generation=1}
   [junit4]   2> 548947 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 548947 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@10ffc2d[collection1] main]
   [junit4]   2> 548947 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/control_collection]
   [junit4]   2> 548948 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.c.ZkStateReader path=[/collections/control_collection] [configName]=[conf1] specified config exists in ZooKeeper
   [junit4]   2> 548948 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 548948 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 548948 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {}
   [junit4]   2> 548948 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 548948 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 548948 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 548949 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 548949 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 548950 INFO  (searcherExecutor-6883-thread-1-processing-n:127.0.0.1:34306_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@10ffc2d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 548950 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Looking up max value of version field to seed version buckets
   [junit4]   2> 548950 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.VersionInfo Refreshing highest value of _version_ for 65536 version buckets from index
   [junit4]   2> 548950 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.VersionInfo No terms found for _version_, cannot seed version bucket highest value from index
   [junit4]   2> 548950 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1536188754652299264
   [junit4]   2> 548952 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Took 1.0ms to seed version buckets with highest version 1536188754652299264
   [junit4]   2> 548952 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController watch zkdir /configs/conf1
   [junit4]   2> 548952 INFO  (coreLoadExecutor-6882-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.CoreContainer registering core: collection1
   [junit4]   2> 548952 INFO  (coreZkRegister-6876-thread-1-processing-n:127.0.0.1:34306_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController Register replica - core:collection1 address:http://127.0.0.1:34306 collection:control_collection shard:shard1
   [junit4]   2> 548953 INFO  (coreZkRegister-6876-thread-1-processing-n:127.0.0.1:34306_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 548955 INFO  (coreZkRegister-6876-thread-1-processing-n:127.0.0.1:34306_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.LeaderElector Joined leadership election with path: /collections/control_collection/leader_elect/shard1/election/96011797137522692-core_node1-n_0000000000
   [junit4]   2> 548955 INFO  (coreZkRegister-6876-thread-1-processing-n:127.0.0.1:34306_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Running the leader process for shard=shard1 and weAreReplacement=false and leaderVoteWait=10000
   [junit4]   2> 548956 INFO  (coreZkRegister-6876-thread-1-processing-n:127.0.0.1:34306_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 548956 INFO  (coreZkRegister-6876-thread-1-processing-n:127.0.0.1:34306_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 548956 INFO  (coreZkRegister-6876-thread-1-processing-n:127.0.0.1:34306_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:34306/collection1/
   [junit4]   2> 548956 INFO  (coreZkRegister-6876-thread-1-processing-n:127.0.0.1:34306_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 548956 INFO  (coreZkRegister-6876-thread-1-processing-n:127.0.0.1:34306_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:34306/collection1/ has no replicas
   [junit4]   2> 548957 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"leader",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection"} current state version: 1
   [junit4]   2> 548957 INFO  (coreZkRegister-6876-thread-1-processing-n:127.0.0.1:34306_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 548959 INFO  (coreZkRegister-6876-thread-1-processing-n:127.0.0.1:34306_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/96011797137522692-core_node1-n_0000000000
   [junit4]   2> 548960 INFO  (coreZkRegister-6876-thread-1-processing-n:127.0.0.1:34306_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:34306/collection1/ shard1
   [junit4]   2> 548960 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"leader",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "base_url":"http://127.0.0.1:34306",
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "state":"active"} current state version: 1
   [junit4]   2> 549062 INFO  (zkCallback-23995-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 549111 INFO  (coreZkRegister-6876-thread-1-processing-n:127.0.0.1:34306_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController We are http://127.0.0.1:34306/collection1/ and leader is http://127.0.0.1:34306/collection1/
   [junit4]   2> 549111 INFO  (coreZkRegister-6876-thread-1-processing-n:127.0.0.1:34306_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:34306
   [junit4]   2> 549111 INFO  (coreZkRegister-6876-thread-1-processing-n:127.0.0.1:34306_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 549111 INFO  (coreZkRegister-6876-thread-1-processing-n:127.0.0.1:34306_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController publishing state=active
   [junit4]   2> 549111 INFO  (coreZkRegister-6876-thread-1-processing-n:127.0.0.1:34306_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:34306_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
   [junit4]   2> 549112 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "core_node_name":"core_node1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:34306",
   [junit4]   2>   "node_name":"127.0.0.1:34306_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"} current state version: 2
   [junit4]   2> 549112 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "core_node_name":"core_node1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:34306",
   [junit4]   2>   "node_name":"127.0.0.1:34306_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 549214 INFO  (zkCallback-23995-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 549308 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.s.i.CloudSolrClient Final constructed zkHost string: 127.0.0.1:46768/solr
   [junit4]   2> 549308 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 549309 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 549310 INFO  (zkCallback-23999-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@85c3e3 name:ZooKeeperConnection Watcher:127.0.0.1:46768/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 549310 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 549310 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 549310 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 549311 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 549311 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 549312 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 549312 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
   [junit4]   2> 549312 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 549312 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 549313 INFO  (zkCallback-24000-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@14be880 name:ZooKeeperConnection Watcher:127.0.0.1:46768/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 549313 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 549314 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 549315 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"create",
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "stateFormat":"2"} current state version: 3
   [junit4]   2> 549315 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
   [junit4]   2> 549389 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/shard-1-001/cores/collection1
   [junit4]   2> 549390 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/shard-1-001
   [junit4]   2> 549390 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 549391 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@b01240{/,null,AVAILABLE}
   [junit4]   2> 549392 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.e.j.s.ServerConnector Started ServerConnector@14c329b{HTTP/1.1,[http/1.1]}{127.0.0.1:33640}
   [junit4]   2> 549392 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.e.j.s.Server Started @550940ms
   [junit4]   2> 549392 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=33640, coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/shard-1-001/cores}
   [junit4]   2> 549393 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@1d16e93
   [junit4]   2> 549393 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/shard-1-001'
   [junit4]   2> 549393 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 549393 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 549393 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 549394 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 549394 INFO  (zkCallback-24001-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@ef80bd name:ZooKeeperConnection Watcher:127.0.0.1:46768/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 549394 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 549395 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 549395 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 549395 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/shard-1-001/solr.xml
   [junit4]   2> 549400 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/shard-1-001/cores
   [junit4]   2> 549400 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.CoreContainer New CoreContainer 3932233
   [junit4]   2> 549400 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/shard-1-001]
   [junit4]   2> 549401 WARN  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.CoreContainer Couldn't add files from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/shard-1-001/lib to classpath: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/shard-1-001/lib
   [junit4]   2> 549401 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 549403 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 549404 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 549404 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 549404 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46768/solr
   [junit4]   2> 549404 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 549404 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 549405 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 549405 INFO  (zkCallback-24004-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4fc37c name:ZooKeeperConnection Watcher:127.0.0.1:46768 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 549405 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 549406 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 549407 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$69/14195583@1b6f3ae
   [junit4]   2> 549407 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 549408 INFO  (zkCallback-24005-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@a2cb1f name:ZooKeeperConnection Watcher:127.0.0.1:46768/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 549408 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 549413 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 549413 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 549415 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 549416 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 549416 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96011797137522697-127.0.0.1:33640_-n_0000000001
   [junit4]   2> 549417 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/96011797137522692-127.0.0.1:34306_-n_0000000000 to know if I could be the leader
   [junit4]   2> 549417 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.c.ZkController Publish node=127.0.0.1:33640_ as DOWN
   [junit4]   2> 549417 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection1/state.json
   [junit4]   2> 549418 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33640_
   [junit4]   2> 549418 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:33640_
   [junit4]   2> 549419 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"downnode",
   [junit4]   2>   "node_name":"127.0.0.1:33640_"} current state version: 3
   [junit4]   2> 549419 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:33640_
   [junit4]   2> 549419 INFO  (zkCallback-23999-thread-1) [    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 549419 INFO  (zkCallback-23995-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 549420 INFO  (zkCallback-23999-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 549420 INFO  (zkCallback-24005-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 549421 INFO  (zkCallback-24005-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 549421 INFO  (zkCallback-23995-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 549421 INFO  (zkCallback-24005-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [2])
   [junit4]   2> 549421 INFO  (zkCallback-23999-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [2])
   [junit4]   2> 549422 INFO  (zkCallback-23995-thread-1-processing-n:127.0.0.1:34306_) [n:127.0.0.1:34306_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [2])
   [junit4]   2> 549454 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 549454 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 549455 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/shard-1-001/cores
   [junit4]   2> 549456 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=collection1, coreNodeName=, dataDir=data/, shard=}
   [junit4]   2> 549456 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.c.CorePropertiesLocator Found core collection1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/shard-1-001/cores/collection1
   [junit4]   2> 549456 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions
   [junit4]   2> 549456 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.s.SolrDispatchFilter user.dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 549456 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [n:127.0.0.1:33640_    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 549457 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 549457 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
   [junit4]   2> 549457 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.ZkController Registering watch for collection collection1
   [junit4]   2> 549458 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:33640",
   [junit4]   2>   "node_name":"127.0.0.1:33640_",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "operation":"state"} current state version: 4
   [junit4]   2> 549458 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.o.ReplicaMutator Update state numShards=2 message={
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:33640",
   [junit4]   2>   "node_name":"127.0.0.1:33640_",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 549458 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.c.ZkStateReader Add data for [collection1] ver [0]
   [junit4]   2> 549458 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.o.ReplicaMutator Collection already exists with numShards=2
   [junit4]   2> 549458 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
   [junit4]   2> 549458 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.ZkController Waiting for coreNodeName for core collection1 in collection collection1 to be assigned
   [junit4]   2> 549559 INFO  (OverseerStateUpdate-96011797137522692-127.0.0.1:34306_-n_0000000000) [n:127.0.0.1:34306_    ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 549560 INFO  (zkCallback-24005-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 549560 INFO  (zkCallback-24005-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_    ] o.a.s.c.c.ZkStateReader Updating data for [collection1] from [0] to [1]
   [junit4]   2> 549560 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection1
   [junit4]   2> 549560 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.ZkController Check for collection zkNode:collection1
   [junit4]   2> 549561 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.ZkController Collection zkNode exists
   [junit4]   2> 549561 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/collection1]
   [junit4]   2> 549561 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.c.ZkStateReader path=[/collections/collection1] [configName]=[conf1] specified config exists in ZooKeeper
   [junit4]   2> 549562 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/shard-1-001/cores/collection1'
   [junit4]   2> 549562 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 549562 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 549563 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.Config loaded config solrconfig.xml with version 0 
   [junit4]   2> 549567 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 549572 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.1.0
   [junit4]   2> 549591 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 549597 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 549697 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.s.IndexSchema default search field in schema is text
   [junit4]   2> 549698 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 549698 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 549702 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 549716 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 549716 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 549716 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 549716 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/shard-1-001/cores/collection1], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/shard-1-001/cores/collection1/data/]
   [junit4]   2> 549716 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@198a54e
   [junit4]   2> 549717 INFO  (coreLoadExecutor-6893-thread-1-processing-n:127.0.0.1:33640_) [n:127.0.0.1:33640_ c:collection1 s:shard2 r:cor

[...truncated too long message...]

irectoryFactory Closing directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001/shard-4-001/cores/collection1/data/index
   [junit4]   2> 649380 INFO  (zkCallback-24024-thread-3-processing-n:127.0.0.1:43873_) [n:127.0.0.1:43873_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.ElectionContext Canceling election /collections/collection1/leader_elect/shard1/election/96011797137522706-core_node4-n_0000000001
   [junit4]   2> 649380 WARN  (zkCallback-24024-thread-3-processing-n:127.0.0.1:43873_) [n:127.0.0.1:43873_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.LeaderElector 
   [junit4]   2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election/96011797137522706-core_node4-n_0000000001
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:244)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:241)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:241)
   [junit4]   2> 	at org.apache.solr.cloud.ElectionContext.cancelElection(ElectionContext.java:86)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContextBase.cancelElection(ElectionContext.java:143)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.cancelElection(ElectionContext.java:279)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:455)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 649381 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.e.j.s.ServerConnector Stopped ServerConnector@134d91c{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 649381 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1c540d4{/,null,UNAVAILABLE}
   [junit4]   2> 649382 INFO  (TEST-TestRequestStatusCollectionAPI.test-seed#[C4F558B7B5850C5E]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46768 46768
   [junit4]   2> 649473 INFO  (Thread-1774) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46768 46768
   [junit4]   2> 649474 WARN  (Thread-1774) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/aliases.json
   [junit4]   2> 	6	/solr/clusterprops.json
   [junit4]   2> 	5	/solr/security.json
   [junit4]   2> 	5	/solr/configs/conf1
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/clusterstate.json
   [junit4]   2> 	4	/solr/collections/collection1/state.json
   [junit4]   2> 	2	/solr/collections/collection2/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/live_nodes
   [junit4]   2> 	6	/solr/collections
   [junit4]   2> 	4	/solr/overseer/collection-queue-work
   [junit4]   2> 	2	/solr/overseer/queue
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestRequestStatusCollectionAPI -Dtests.method=test -Dtests.seed=C4F558B7B5850C5E -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ja-JP-u-ca-japanese-x-lvariant-JP -Dtests.timezone=Asia/Ho_Chi_Minh -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE  101s J0 | TestRequestStatusCollectionAPI.test <<<
   [junit4]    > Throwable #1: org.junit.ComparisonFailure: expected:<found [1001] in [completed] tasks> but was:<found [1001] in [running] tasks>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C4F558B7B5850C5E:4CA1676D1B7961A6]:0)
   [junit4]    > 	at org.apache.solr.cloud.TestRequestStatusCollectionAPI.test(TestRequestStatusCollectionAPI.java:111)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 649479 INFO  (SUITE-TestRequestStatusCollectionAPI-seed#[C4F558B7B5850C5E]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.TestRequestStatusCollectionAPI_C4F558B7B5850C5E-001
   [junit4]   2> Jun 04, 2016 7:04:20 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene60): {}, docValues:{}, maxPointsInLeafNode=325, maxMBSortInHeap=5.432560298537687, sim=ClassicSimilarity, locale=ja-JP-u-ca-japanese-x-lvariant-JP, timezone=Asia/Ho_Chi_Minh
   [junit4]   2> NOTE: Linux 4.4.0-21-generic i386/Oracle Corporation 1.8.0_92 (32-bit)/cpus=12,threads=1,free=56083912,total=191025152
   [junit4]   2> NOTE: All tests run in this JVM: [TestImplicitCoreProperties, SOLR749Test, TestHighlightDedupGrouping, TestXmlQParser, TestBinaryField, DistributedFacetPivotWhiteBoxTest, BasicZkTest, TestMaxScoreQueryParser, TestSubQueryTransformerCrossCore, TestSchemaNameResource, TestConfig, CurrencyFieldOpenExchangeTest, ZkNodePropsTest, ForceLeaderTest, TestArbitraryIndexDir, BigEndianAscendingWordSerializerTest, TestFaceting, TestStressLiveNodes, PingRequestHandlerTest, SolrCloudExampleTest, TestLMDirichletSimilarityFactory, TestDefaultStatsCache, LeaderElectionIntegrationTest, TestDynamicFieldCollectionResource, URLClassifyProcessorTest, AlternateDirectoryTest, SynonymTokenizerTest, TestValueSourceCache, SmileWriterTest, PreAnalyzedFieldTest, ConnectionReuseTest, TestManagedResourceStorage, TestRangeQuery, TestSolrConfigHandler, TestExpandComponent, BlockJoinFacetDistribTest, TestManagedSchema, TestPseudoReturnFields, TestDocSet, TestExactStatsCache, TestGraphTermsQParserPlugin, TestClassicSimilarityFactory, SystemInfoHandlerTest, RAMDirectoryFactoryTest, ReturnFieldsTest, FullHLLTest, DeleteLastCustomShardedReplicaTest, CopyFieldTest, TestReplicationHandlerBackup, TestJoin, LeaderInitiatedRecoveryOnCommitTest, HdfsDirectoryTest, CdcrVersionReplicationTest, XsltUpdateRequestHandlerTest, TestManagedSynonymFilterFactory, VersionInfoTest, UUIDFieldTest, TestOverriddenPrefixQueryForCustomFieldType, HdfsChaosMonkeySafeLeaderTest, TestSizeLimitedDistributedMap, TestFieldResource, PrimUtilsTest, JsonLoaderTest, TestReloadDeadlock, DistributedSuggestComponentTest, TestLuceneMatchVersion, DocumentAnalysisRequestHandlerTest, SSLMigrationTest, TestRemoteStreaming, CollectionTooManyReplicasTest, TestDocBasedVersionConstraints, DisMaxRequestHandlerTest, DistributedQueryElevationComponentTest, AnalysisAfterCoreReloadTest, QueryResultKeyTest, GraphQueryTest, AsyncCallRequestStatusResponseTest, FileBasedSpellCheckerTest, ActionThrottleTest, TestSchemaVersionResource, TestRequestStatusCollectionAPI]
   [junit4] Completed [135/613 (1!)] on J0 in 101.24s, 1 test, 1 failure <<< FAILURES!

[...truncated 1558 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 1 failure, 115 ignored (71 assumptions) [seed: C4F558B7B5850C5E]

Total time: 65 minutes 30 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS-EA] Lucene-Solr-6.x-Linux (64bit/jdk-9-ea+121) - Build # 826 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/826/
Java: 64bit/jdk-9-ea+121 -XX:-UseCompressedOops -XX:+UseParallelGC

2 tests failed.
FAILED:  org.apache.solr.cloud.overseer.ZkStateReaderTest.testStateFormatUpdateWithTimeDelay

Error Message:


Stack Trace:
java.util.concurrent.TimeoutException
	at __randomizedtesting.SeedInfo.seed([86E3783F3BC6DADD:F97DCFBA52A4F757]:0)
	at org.apache.solr.common.cloud.ZkStateReader.waitForState(ZkStateReader.java:1200)
	at org.apache.solr.cloud.overseer.ZkStateReaderTest.testStateFormatUpdate(ZkStateReaderTest.java:126)
	at org.apache.solr.cloud.overseer.ZkStateReaderTest.testStateFormatUpdateWithTimeDelay(ZkStateReaderTest.java:54)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.handler.TestReqParamsAPI.test

Error Message:
Could not get expected value  'null' for path 'response/params/y/p' full output: {   "responseHeader":{     "status":0,     "QTime":0},   "response":{     "znodeVersion":4,     "params":{       "x":{         "a":"A val",         "b":"B val",         "_appends_":{"add":"first"},         "_invariants_":{"fixed":"f"},         "":{"v":1}},       "y":{         "p":"P val",         "q":"Q val",         "":{"v":2}}}}},  from server:  http://127.0.0.1:38673/collection1

Stack Trace:
java.lang.AssertionError: Could not get expected value  'null' for path 'response/params/y/p' full output: {
  "responseHeader":{
    "status":0,
    "QTime":0},
  "response":{
    "znodeVersion":4,
    "params":{
      "x":{
        "a":"A val",
        "b":"B val",
        "_appends_":{"add":"first"},
        "_invariants_":{"fixed":"f"},
        "":{"v":1}},
      "y":{
        "p":"P val",
        "q":"Q val",
        "":{"v":2}}}}},  from server:  http://127.0.0.1:38673/collection1
	at __randomizedtesting.SeedInfo.seed([86E3783F3BC6DADD:EB747E5953AB725]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.core.TestSolrConfigHandler.testForResponseElement(TestSolrConfigHandler.java:457)
	at org.apache.solr.handler.TestReqParamsAPI.testReqParams(TestReqParamsAPI.java:259)
	at org.apache.solr.handler.TestReqParamsAPI.test(TestReqParamsAPI.java:62)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)




Build Log:
[...truncated 11278 lines...]
   [junit4] Suite: org.apache.solr.cloud.overseer.ZkStateReaderTest
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.overseer.ZkStateReaderTest_86E3783F3BC6DADD-001/init-core-data-001
   [junit4]   2> 495044 INFO  (SUITE-ZkStateReaderTest-seed#[86E3783F3BC6DADD]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 495045 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testStateFormatUpdateWithExplicitRefresh
   [junit4]   2> 495045 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 495045 INFO  (Thread-1665) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 495045 INFO  (Thread-1665) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 495145 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer start zk server on port:44916
   [junit4]   2> 495145 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 495146 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 495147 INFO  (zkCallback-966-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@76b993d9 name:ZooKeeperConnection Watcher:127.0.0.1:44916 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495147 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 495148 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 495148 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 495148 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 495149 INFO  (zkCallback-967-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2fb12867 name:ZooKeeperConnection Watcher:127.0.0.1:44916 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495149 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 495150 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 495150 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 495151 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 495152 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 495152 INFO  (zkCallback-968-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@57ec51b2 name:ZooKeeperConnection Watcher:127.0.0.1:44916/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495153 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 495153 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 495153 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 495153 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 495154 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 495154 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 495154 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 495155 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 495155 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 495155 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 495156 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Deleting data for [c1]
   [junit4]   2> 495156 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 495163 INFO  (zkCallback-968-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 495164 INFO  (zkCallback-968-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 495164 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 495164 INFO  (zkCallback-968-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/c1/state.json] for collection [c1] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 495165 INFO  (zkCallback-968-thread-1) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 495165 INFO  (zkCallback-968-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 495166 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44916 44916
   [junit4]   2> 495208 INFO  (Thread-1665) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44916 44916
   [junit4]   2> 495208 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[86E3783F3BC6DADD]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStateFormatUpdateWithExplicitRefresh
   [junit4]   2> 495210 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testWatchedCollectionCreation
   [junit4]   2> 495210 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 495210 INFO  (Thread-1666) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 495210 INFO  (Thread-1666) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 495310 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer start zk server on port:44788
   [junit4]   2> 495311 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 495311 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 495312 INFO  (zkCallback-970-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@21c82f42 name:ZooKeeperConnection Watcher:127.0.0.1:44788 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495312 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 495312 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 495313 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 495313 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 495314 INFO  (zkCallback-971-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@155c9349 name:ZooKeeperConnection Watcher:127.0.0.1:44788 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495314 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 495314 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 495314 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 495315 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15519c4e3010001, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]   2> 495315 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 495316 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 495317 INFO  (zkCallback-972-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3920a87c name:ZooKeeperConnection Watcher:127.0.0.1:44788/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495317 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 495317 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 495317 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 495318 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 495318 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 495318 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 495319 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 495319 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 495319 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 495319 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 495320 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Deleting data for [c1]
   [junit4]   2> 495320 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 495320 INFO  (zkCallback-972-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 495321 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 495321 INFO  (zkCallback-972-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/c1/state.json] for collection [c1] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 495321 INFO  (zkCallback-972-thread-1) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 495322 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44788 44788
   [junit4]   2> 495446 INFO  (Thread-1666) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44788 44788
   [junit4]   2> 495446 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[86E3783F3BC6DADD]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testWatchedCollectionCreation
   [junit4]   2> 495447 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testStateFormatUpdateWithTimeDelayLazy
   [junit4]   2> 495448 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 495448 INFO  (Thread-1667) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 495448 INFO  (Thread-1667) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 495548 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer start zk server on port:34339
   [junit4]   2> 495548 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 495549 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 495550 INFO  (zkCallback-974-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3914652b name:ZooKeeperConnection Watcher:127.0.0.1:34339 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495550 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 495550 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 495550 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 495551 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 495551 INFO  (zkCallback-975-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7959410c name:ZooKeeperConnection Watcher:127.0.0.1:34339 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495551 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 495551 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 495551 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 495552 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 495552 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 495553 INFO  (zkCallback-976-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6b7cc039 name:ZooKeeperConnection Watcher:127.0.0.1:34339/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495553 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 495553 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 495553 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 495554 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 495554 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 495555 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 495555 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 495555 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 495555 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 495556 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 495556 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 495556 INFO  (zkCallback-976-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 495557 INFO  (zkCallback-976-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 495557 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Deleting data for [c1]
   [junit4]   2> 495558 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 495558 INFO  (zkCallback-976-thread-1) [    ] o.a.s.c.c.ZkStateReader Uninteresting collection c1
   [junit4]   2> 495558 INFO  (zkCallback-976-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 495559 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 495559 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34339 34339
   [junit4]   2> 495590 INFO  (Thread-1667) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34339 34339
   [junit4]   2> 495590 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStateFormatUpdateWithTimeDelayLazy
   [junit4]   2> 495592 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testExternalCollectionWatchedNotWatched
   [junit4]   2> 495592 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 495592 INFO  (Thread-1668) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 495592 INFO  (Thread-1668) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 495692 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer start zk server on port:35663
   [junit4]   2> 495692 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 495693 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 495694 INFO  (zkCallback-978-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3dbfd075 name:ZooKeeperConnection Watcher:127.0.0.1:35663 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495694 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 495694 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 495694 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 495695 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 495695 INFO  (zkCallback-979-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@22900b14 name:ZooKeeperConnection Watcher:127.0.0.1:35663 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495695 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 495695 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 495695 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 495696 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 495696 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 495697 INFO  (zkCallback-980-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@11bbefea name:ZooKeeperConnection Watcher:127.0.0.1:35663/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495697 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 495697 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 495697 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 495698 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 495698 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 495698 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 495699 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 495699 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 495699 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 495699 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 495699 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 495700 INFO  (zkCallback-980-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 495700 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 495700 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 495701 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35663 35663
   [junit4]   2> 495718 INFO  (Thread-1668) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35663 35663
   [junit4]   2> 495719 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[86E3783F3BC6DADD]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testExternalCollectionWatchedNotWatched
   [junit4]   2> 495720 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testStateFormatUpdateWithTimeDelay
   [junit4]   2> 495720 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 495720 INFO  (Thread-1669) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 495720 INFO  (Thread-1669) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 495820 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer start zk server on port:45899
   [junit4]   2> 495821 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 495822 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 495822 INFO  (zkCallback-982-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@72fed53e name:ZooKeeperConnection Watcher:127.0.0.1:45899 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495822 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 495823 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 495823 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 495823 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 495824 INFO  (zkCallback-983-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3e401e1f name:ZooKeeperConnection Watcher:127.0.0.1:45899 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495824 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 495824 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 495824 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 495824 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 495825 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 495825 INFO  (zkCallback-984-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3587b0e3 name:ZooKeeperConnection Watcher:127.0.0.1:45899/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495825 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 495825 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 495826 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 495826 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 495826 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 495827 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 495827 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 495828 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 495828 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 495828 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 495828 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Deleting data for [c1]
   [junit4]   2> 495828 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 495829 INFO  (zkCallback-984-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 495830 INFO  (zkCallback-984-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 495830 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 495831 INFO  (zkCallback-984-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/c1/state.json] for collection [c1] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 495831 INFO  (zkCallback-984-thread-1) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 495831 INFO  (zkCallback-984-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 525832 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45899 45899
   [junit4]   2> 525852 INFO  (Thread-1669) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45899 45899
   [junit4]   2> 525852 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[86E3783F3BC6DADD]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStateFormatUpdateWithTimeDelay
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ZkStateReaderTest -Dtests.method=testStateFormatUpdateWithTimeDelay -Dtests.seed=86E3783F3BC6DADD -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=en-CA -Dtests.timezone=Etc/GMT0 -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   30.1s J1 | ZkStateReaderTest.testStateFormatUpdateWithTimeDelay <<<
   [junit4]    > Throwable #1: java.util.concurrent.TimeoutException
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([86E3783F3BC6DADD:F97DCFBA52A4F757]:0)
   [junit4]    > 	at org.apache.solr.common.cloud.ZkStateReader.waitForState(ZkStateReader.java:1200)
   [junit4]    > 	at org.apache.solr.cloud.overseer.ZkStateReaderTest.testStateFormatUpdate(ZkStateReaderTest.java:126)
   [junit4]    > 	at org.apache.solr.cloud.overseer.ZkStateReaderTest.testStateFormatUpdateWithTimeDelay(ZkStateReaderTest.java:54)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]   2> 525855 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testStateFormatUpdateWithExplicitRefreshLazy
   [junit4]   2> 525856 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 525856 INFO  (Thread-1670) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 525856 INFO  (Thread-1670) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 525956 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer start zk server on port:45355
   [junit4]   2> 525956 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 525956 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 525957 INFO  (zkCallback-986-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2d936696 name:ZooKeeperConnection Watcher:127.0.0.1:45355 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 525958 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 525958 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 525958 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 525958 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 525959 INFO  (zkCallback-987-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5c478ca1 name:ZooKeeperConnection Watcher:127.0.0.1:45355 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 525959 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 525959 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 525959 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 525960 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 525960 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 525960 INFO  (zkCallback-988-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6fdef652 name:ZooKeeperConnection Watcher:127.0.0.1:45355/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 525960 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 525961 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 525961 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 525961 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 525962 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 525962 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 525963 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 525963 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 525963 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 525963 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 525964 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 525964 INFO  (zkCallback-988-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 525964 INFO  (zkCallback-988-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 525965 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 525965 INFO  (zkCallback-988-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 525966 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45355 45355
   [junit4]   2> 526074 INFO  (Thread-1670) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45355 45355
   [junit4]   2> 526074 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[86E3783F3BC6DADD]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStateFormatUpdateWithExplicitRefreshLazy
   [junit4]   2> 526075 INFO  (SUITE-ZkStateReaderTest-seed#[86E3783F3BC6DADD]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.overseer.ZkStateReaderTest_86E3783F3BC6DADD-001
   [junit4]   2> Jun 04, 2016 4:56:52 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene60): {}, docValues:{}, maxPointsInLeafNode=1304, maxMBSortInHeap=5.51947983750988, sim=ClassicSimilarity, locale=en-CA, timezone=Etc/GMT0
   [junit4]   2> NOTE: Linux 4.4.0-21-generic amd64/Oracle Corporation 9-ea (64-bit)/cpus=12,threads=1,free=239959304,total=521142272
   [junit4]   2> NOTE: All tests run in this JVM: [TestScoreJoinQPNoScore, TestRawTransformer, TestSolrConfigHandlerCloud, SolrIndexSplitterTest, TestStressVersions, TestObjectReleaseTracker, DistributedQueueTest, TestLFUCache, TestAnalyzeInfixSuggestions, SpellCheckCollatorTest, TestExceedMaxTermLength, TestFreeTextSuggestions, DocValuesMissingTest, DistributedMLTComponentTest, BlockDirectoryTest, ConfigSetsAPITest, TestFieldCacheReopen, TestSearcherReuse, TestFieldTypeCollectionResource, TestLRUStatsCache, TestUpdate, CustomCollectionTest, MultiTermTest, TestTestInjection, TestLegacyFieldCache, TestGeoJSONResponseWriter, TermsComponentTest, SortByFunctionTest, TestCodecSupport, CursorMarkTest, TestTolerantUpdateProcessorCloud, TestCloudDeleteByQuery, TestPerFieldSimilarityWithDefaultOverride, OpenExchangeRatesOrgProviderTest, TestBM25SimilarityFactory, TermVectorComponentTest, TestPartialUpdateDeduplication, DocumentBuilderTest, TestSolrDeletionPolicy2, BasicFunctionalityTest, TestPivotHelperCode, ZkControllerTest, TestJsonFacets, SpellingQueryConverterTest, HdfsThreadLeakTest, TestBlobHandler, OpenCloseCoreStressTest, ZkCLITest, CollectionsAPIAsyncDistributedZkTest, DistributedIntervalFacetingTest, CdcrReplicationHandlerTest, MinimalSchemaTest, TestSubQueryTransformerDistrib, CloudMLTQParserTest, LeaderInitiatedRecoveryOnShardRestartTest, TestHashPartitioner, IndexBasedSpellCheckerTest, FileUtilsTest, TestSerializedLuceneMatchVersion, WordBreakSolrSpellCheckerTest, TestCollectionAPI, LoggingHandlerTest, BitVectorTest, HdfsRecoveryZkTest, MBeansHandlerTest, TestIntervalFaceting, ZkStateReaderTest]
   [junit4] Completed [235/613 (1!)] on J1 in 31.79s, 6 tests, 1 error <<< FAILURES!

[...truncated 889 lines...]
   [junit4] Suite: org.apache.solr.handler.TestReqParamsAPI
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/init-core-data-001
   [junit4]   2> 1519578 INFO  (SUITE-TestReqParamsAPI-seed#[86E3783F3BC6DADD]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1519578 INFO  (SUITE-TestReqParamsAPI-seed#[86E3783F3BC6DADD]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 1519579 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1519579 INFO  (Thread-3966) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1519579 INFO  (Thread-3966) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1519679 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer start zk server on port:38757
   [junit4]   2> 1519680 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1519680 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1519681 INFO  (zkCallback-2201-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6d823b8 name:ZooKeeperConnection Watcher:127.0.0.1:38757 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1519681 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1519682 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1519682 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 1519683 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1519684 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1519685 INFO  (zkCallback-2202-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@60925b77 name:ZooKeeperConnection Watcher:127.0.0.1:38757/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1519685 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1519685 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1519685 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 1519686 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards
   [junit4]   2> 1519686 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection
   [junit4]   2> 1519686 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards
   [junit4]   2> 1519687 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1519687 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1519688 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1519688 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
   [junit4]   2> 1519688 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1519688 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1519689 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1519690 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1519690 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1519691 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1519691 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1519691 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml
   [junit4]   2> 1519692 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1519692 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1519692 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1519692 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1519693 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1519693 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1519694 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1519694 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1519694 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1519694 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1519766 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/control-001/cores/collection1
   [junit4]   2> 1519767 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 1519767 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3d17ded6{/,null,AVAILABLE}
   [junit4]   2> 1519768 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.e.j.s.ServerConnector Started ServerConnector@5e352432{HTTP/1.1,[http/1.1]}{127.0.0.1:37913}
   [junit4]   2> 1519768 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.e.j.s.Server Started @1521770ms
   [junit4]   2> 1519768 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/, solr.data.dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/tempDir-001/control/data, hostPort=37913, coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/control-001/cores}
   [junit4]   2> 1519768 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): jdk.internal.loader.ClassLoaders$AppClassLoader@71e7a66b
   [junit4]   2> 1519768 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/control-001'
   [junit4]   2> 1519768 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1519768 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 1519768 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1519769 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1519769 INFO  (zkCallback-2203-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2962094d name:ZooKeeperConnection Watcher:127.0.0.1:38757/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1519769 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1519770 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1519770 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1519770 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/control-001/solr.xml
   [junit4]   2> 1519774 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/control-001/cores
   [junit4]   2> 1519774 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.CoreContainer New CoreContainer 396780164
   [junit4]   2> 1519775 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/control-001]
   [junit4]   2> 1519775 WARN  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.CoreContainer Couldn't add files from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/control-001/lib to classpath: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/control-001/lib
   [junit4]   2> 1519775 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 1519775 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1519776 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1519776 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1519776 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38757/solr
   [junit4]   2> 1519776 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 1519776 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 1519776 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1519777 INFO  (zkCallback-2206-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2b363a7a name:ZooKeeperConnection Watcher:127.0.0.1:38757 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1519777 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1519777 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 1519777 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$190/510460054@26d317ca
   [junit4]   2> 1519778 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1519778 INFO  (zkCallback-2207-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@244c6477 name:ZooKeeperConnection Watcher:127.0.0.1:38757/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1519778 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1519779 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 1519779 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 1519780 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 1519781 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 1519782 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 1519782 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 1519783 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 1519783 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 1519783 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 1519784 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 1519784 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 1519784 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 1519785 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 1519786 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 1519787 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1519787 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96011367610712068-127.0.0.1:37913_-n_0000000000
   [junit4]   2> 1519787 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37913_
   [junit4]   2> 1519788 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 1519788 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.Overseer Overseer (id=96011367610712068-127.0.0.1:37913_-n_0000000000) starting
   [junit4]   2> 1519789 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 1519791 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1519791 INFO  (OverseerCollectionConfigSetProcessor-96011367610712068-127.0.0.1:37913_-n_0000000000) [    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 1519791 INFO  (OverseerStateUpdate-96011367610712068-127.0.0.1:37913_-n_0000000000) [    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 1519791 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37913_
   [junit4]   2> 1519792 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:37913_
   [junit4]   2> 1519792 INFO  (OverseerStateUpdate-96011367610712068-127.0.0.1:37913_-n_0000000000) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 1519792 INFO  (zkCallback-2207-thread-1) [    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 1519792 INFO  (zkCallback-2207-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1519805 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 1519805 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 1519805 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/control-001/cores
   [junit4]   2> 1519806 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {schema=schema.xml, dataDir=data/, collection=control_collection, loadOnStartup=true, coreNodeName=, transient=false, name=collection1, shard=, config=solrconfig.xml, configSetProperties=configsetprops.json}
   [junit4]   2> 1519806 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.CorePropertiesLocator Found core collection1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/control-001/cores/collection1
   [junit4]   2> 1519806 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions
   [junit4]   2> 1519806 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 1519806 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
   [junit4]   2> 1519806 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.s.SolrDispatchFilter user.dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2
   [junit4]   2> 1519806 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 1519807 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.ZkController Collection control_collection not visible yet, but flagging it so a watch is registered when it becomes visible
   [junit4]   2> 1519807 INFO  (OverseerStateUpdate-96011367610712068-127.0.0.1:37913_-n_0000000000) [    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:37913",
   [junit4]   2>   "node_name":"127.0.0.1:37913_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 1519807 INFO  (OverseerStateUpdate-96011367610712068-127.0.0.1:37913_-n_0000000000) [    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:37913",
   [junit4]   2>   "node_name":"127.0.0.1:37913_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 1519807 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.c.ZkStateReader Deleting data for [control_collection]
   [junit4]   2> 1519807 INFO  (OverseerStateUpdate-96011367610712068-127.0.0.1:37913_-n_0000000000) [    ] o.a.s.c.o.ClusterStateMutator building a new cName: control_collection
   [junit4]   2> 1519807 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.ZkController Waiting for coreNodeName for core collection1 in collection control_collection to be assigned
   [junit4]   2> 1519807 INFO  (OverseerStateUpdate-96011367610712068-127.0.0.1:37913_-n_0000000000) [    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 1519808 INFO  (zkCallback-2207-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1519808 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1519808 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.ZkController Check for collection zkNode:control_collection
   [junit4]   2> 1519808 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.ZkController Collection zkNode exists
   [junit4]   2> 1519808 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/control_collection]
   [junit4]   2> 1519809 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.c.ZkStateReader path=[/collections/control_collection] [configName]=[conf1] specified config exists in ZooKeeper
   [junit4]   2> 1519809 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/control-001/cores/collection1'
   [junit4]   2> 1519809 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1519809 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 1519811 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.Config loaded config solrconfig.xml with version 0 
   [junit4]   2> 1519813 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 1519816 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.1.0
   [junit4]   2> 1519823 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1519827 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1519900 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.s.OpenExchangeRatesOrgProvider Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1519904 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.s.IndexSchema default search field in schema is text
   [junit4]   2> 1519905 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 1519909 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 1519910 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 1519911 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.s.OpenExchangeRatesOrgProvider Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1519912 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.s.OpenExchangeRatesOrgProvider Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1519912 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2
   [junit4]   2> 1519913 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1519913 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1519913 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/control-001/cores/collection1], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/control-001/cores/collection1/data/]
   [junit4]   2> 1519913 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5c6748ed
   [junit4]   2> 1519913 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/control-001/cores/collection1/data
   [junit4]   2> 1519913 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.SolrCore New index directory detected: old=null new=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1519913 WARN  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.SolrCore [collection1] Solr index directory '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1519914 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001/control-001/cores/collection1/data/index
   [junit4]   2> 1519914 INFO  (coreLoadExecutor-4967-thread-1) [    ] o.a.s.u.RandomMergePolicy Random

[...truncated too long message...]

E3783F3BC6DADD]) [    ] o.a.s.c.Overseer Overseer (id=96011367610712081-127.0.0.1:38673_-n_0000000004) closing
   [junit4]   2> 1537358 INFO  (OverseerStateUpdate-96011367610712081-127.0.0.1:38673_-n_0000000004) [    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:38673_
   [junit4]   2> 1537359 INFO  (zkCallback-2235-thread-2) [    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1537359 WARN  (zkCallback-2235-thread-2) [    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1537360 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.e.j.s.ServerConnector Stopped ServerConnector@64c5620{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1537360 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@f0da1c7{/,null,UNAVAILABLE}
   [junit4]   2> 1537360 INFO  (TEST-TestReqParamsAPI.test-seed#[86E3783F3BC6DADD]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38757 38757
   [junit4]   2> 1537464 INFO  (Thread-3966) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38757 38757
   [junit4]   2> 1537465 WARN  (Thread-3966) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	45	/solr/configs/conf1
   [junit4]   2> 	6	/solr/aliases.json
   [junit4]   2> 	6	/solr/clusterprops.json
   [junit4]   2> 	5	/solr/security.json
   [junit4]   2> 	4	/solr/collections/collection1/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/live_nodes
   [junit4]   2> 	6	/solr/collections
   [junit4]   2> 	4	/solr/overseer/collection-queue-work
   [junit4]   2> 	3	/solr/overseer/queue
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestReqParamsAPI -Dtests.method=test -Dtests.seed=86E3783F3BC6DADD -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=en-SZ -Dtests.timezone=America/Winnipeg -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 17.9s J2 | TestReqParamsAPI.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Could not get expected value  'null' for path 'response/params/y/p' full output: {
   [junit4]    >   "responseHeader":{
   [junit4]    >     "status":0,
   [junit4]    >     "QTime":0},
   [junit4]    >   "response":{
   [junit4]    >     "znodeVersion":4,
   [junit4]    >     "params":{
   [junit4]    >       "x":{
   [junit4]    >         "a":"A val",
   [junit4]    >         "b":"B val",
   [junit4]    >         "_appends_":{"add":"first"},
   [junit4]    >         "_invariants_":{"fixed":"f"},
   [junit4]    >         "":{"v":1}},
   [junit4]    >       "y":{
   [junit4]    >         "p":"P val",
   [junit4]    >         "q":"Q val",
   [junit4]    >         "":{"v":2}}}}},  from server:  http://127.0.0.1:38673/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([86E3783F3BC6DADD:EB747E5953AB725]:0)
   [junit4]    > 	at org.apache.solr.core.TestSolrConfigHandler.testForResponseElement(TestSolrConfigHandler.java:457)
   [junit4]    > 	at org.apache.solr.handler.TestReqParamsAPI.testReqParams(TestReqParamsAPI.java:259)
   [junit4]    > 	at org.apache.solr.handler.TestReqParamsAPI.test(TestReqParamsAPI.java:62)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]   2> 1537467 INFO  (SUITE-TestReqParamsAPI-seed#[86E3783F3BC6DADD]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.handler.TestReqParamsAPI_86E3783F3BC6DADD-001
   [junit4]   2> Jun 04, 2016 5:13:43 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene60): {}, docValues:{}, maxPointsInLeafNode=149, maxMBSortInHeap=7.000327175207677, sim=ClassicSimilarity, locale=en-SZ, timezone=America/Winnipeg
   [junit4]   2> NOTE: Linux 4.4.0-21-generic amd64/Oracle Corporation 9-ea (64-bit)/cpus=12,threads=1,free=26682544,total=510656512
   [junit4]   2> NOTE: All tests run in this JVM: [BJQParserTest, TestSolrQueryResponse, CoreMergeIndexesAdminHandlerTest, HardAutoCommitTest, TestSolrJ, TestFastWriter, OverseerTest, TestInitQParser, TestBlendedInfixSuggestions, ResponseHeaderTest, TestHdfsUpdateLog, TestFuzzyAnalyzedSuggestions, BadIndexSchemaTest, DistributedFacetPivotSmallTest, TestHighFrequencyDictionaryFactory, LeaderFailoverAfterPartitionTest, TestDocTermOrds, TestSearchPerf, TestFieldTypeResource, TestExactSharedStatsCache, TestIndexingPerformance, MigrateRouteKeyTest, CurrencyFieldXmlFileTest, TestUtils, TestFieldCacheVsDocValues, TestWriterPerf, TestMacros, TestPHPSerializedResponseWriter, TestTrackingShardHandlerFactory, TestStressRecovery, TestJmxIntegration, MergeStrategyTest, HdfsNNFailoverTest, TestLockTree, TestSweetSpotSimilarityFactory, PrimitiveFieldTypeTest, TestScoreJoinQPScore, TestDistributedStatsComponentCardinality, FieldMutatingUpdateProcessorTest, AutoCommitTest, FieldAnalysisRequestHandlerTest, SampleTest, HighlighterConfigTest, StressHdfsTest, TestSort, SuggesterTSTTest, PluginInfoTest, XmlUpdateRequestHandlerTest, TestInitParams, HdfsTlogReplayBufferedWhileIndexingTest, TestCoreDiscovery, FullSolrCloudDistribCmdsTest, TestDistributedMissingSort, CollectionsAPISolrJTest, TestGroupingSearch, TestManagedResource, BlockJoinFacetSimpleTest, RemoteQueryErrorTest, TestShortCircuitedRequests, NoCacheHeaderTest, UUIDUpdateProcessorFallbackTest, TestSolrQueryParserDefaultOperatorResource, SuggesterTest, SegmentsInfoRequestHandlerTest, ExplicitHLLTest, HdfsDirectoryFactoryTest, ShowFileRequestHandlerTest, TestGraphMLResponseWriter, SolrCoreCheckLockOnStartupTest, LeaderElectionTest, TestManagedStopFilterFactory, TestExtendedDismaxParser, SliceStateTest, TestNonDefinedSimilarityFactory, ShardSplitTest, RequestHandlersTest, TestDefaultSearchFieldResource, ChangedSchemaMergeTest, ClusterStateTest, OverriddenZkACLAndCredentialsProvidersTest, TestTolerantSearch, TestCollationField, TestRecoveryHdfs, TestTolerantUpdateProcessorRandomCloud, DirectUpdateHandlerTest, RulesTest, ZkStateWriterTest, HLLUtilTest, TestDynamicLoading, TestComplexPhraseQParserPlugin, RegexBoostProcessorTest, TestJettySolrRunner, TestReversedWildcardFilterFactory, DirectUpdateHandlerOptimizeTest, TestSolrXml, AssignTest, TestJmxMonitoredMap, UnloadDistributedZkTest, TestSubQueryTransformer, TestAuthenticationFramework, TestDocSet, ExternalFileFieldSortTest, TestSolrCoreProperties, NotRequiredUniqueKeyTest, SpellPossibilityIteratorTest, TestCharFilters, SynonymTokenizerTest, EchoParamsTest, TestLMJelinekMercerSimilarityFactory, ResourceLoaderTest, URLClassifyProcessorTest, DateMathParserTest, TestSystemIdResolver, PrimUtilsTest, TestSuggestSpellingConverter, DateFieldTest, DOMUtilTest, TestLRUCache, SystemInfoHandlerTest, CircularListTest, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, TestEmbeddedSolrServerConstructors, AsyncMigrateRouteKeyTest, CollectionStateFormat2Test, ConnectionManagerTest, DeleteShardTest, DistribCursorPagingTest, ForceLeaderTest, TestOnReconnectListenerSupport, TestRandomRequestDistribution, TestSolrCloudWithKerberosAlt, TestStressCloudBlindAtomicUpdates, TestInfoStreamLogging, TestReloadAndDeleteDocs, TestShardHandlerFactory, TestSolrConfigHandler, TestReqParamsAPI]
   [junit4] Completed [512/613 (2!)] on J2 in 18.15s, 1 test, 1 failure <<< FAILURES!

[...truncated 344 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 1 error, 1 failure, 118 ignored (72 assumptions) [seed: 86E3783F3BC6DADD]

Total time: 50 minutes 44 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any